2024-11-25 05:54:44,522 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-25 05:54:44,533 main DEBUG Took 0.009388 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-25 05:54:44,534 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-25 05:54:44,534 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-25 05:54:44,535 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-25 05:54:44,536 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 05:54:44,543 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-25 05:54:44,555 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 05:54:44,556 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 05:54:44,556 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 05:54:44,557 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 05:54:44,557 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 05:54:44,557 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 05:54:44,558 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 05:54:44,559 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 05:54:44,559 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 05:54:44,559 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 05:54:44,560 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 05:54:44,560 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 05:54:44,561 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 05:54:44,561 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 05:54:44,562 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 05:54:44,562 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 05:54:44,562 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 05:54:44,562 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 05:54:44,563 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 05:54:44,563 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 05:54:44,564 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 05:54:44,564 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 05:54:44,564 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 05:54:44,564 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 05:54:44,565 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 05:54:44,565 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-25 05:54:44,566 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 05:54:44,568 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-25 05:54:44,569 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-25 05:54:44,570 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-25 05:54:44,571 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-25 05:54:44,571 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-25 05:54:44,578 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-25 05:54:44,580 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-25 05:54:44,582 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-25 05:54:44,583 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-25 05:54:44,583 main DEBUG createAppenders(={Console}) 2024-11-25 05:54:44,584 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-25 05:54:44,584 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-25 05:54:44,584 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-25 05:54:44,585 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-25 05:54:44,585 main DEBUG OutputStream closed 2024-11-25 05:54:44,585 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-25 05:54:44,585 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-25 05:54:44,586 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-25 05:54:44,657 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-25 05:54:44,659 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-25 05:54:44,661 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-25 05:54:44,662 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-25 05:54:44,663 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-25 05:54:44,663 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-25 05:54:44,664 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-25 05:54:44,664 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-25 05:54:44,665 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-25 05:54:44,665 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-25 05:54:44,666 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-25 05:54:44,666 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-25 05:54:44,667 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-25 05:54:44,667 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-25 05:54:44,668 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-25 05:54:44,668 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-25 05:54:44,668 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-25 05:54:44,670 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-25 05:54:44,672 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25 05:54:44,673 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-25 05:54:44,673 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-25 05:54:44,674 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-25T05:54:44,908 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6586672e-597e-41c1-14d9-2bcb3b602f07 2024-11-25 05:54:44,911 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-25 05:54:44,911 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25T05:54:44,918 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-25T05:54:44,939 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-25T05:54:44,962 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6586672e-597e-41c1-14d9-2bcb3b602f07/cluster_8af6f6a5-ddee-228b-0d29-2ed39e53a991, deleteOnExit=true 2024-11-25T05:54:44,962 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-25T05:54:44,964 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6586672e-597e-41c1-14d9-2bcb3b602f07/test.cache.data in system properties and HBase conf 2024-11-25T05:54:44,965 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6586672e-597e-41c1-14d9-2bcb3b602f07/hadoop.tmp.dir in system properties and HBase conf 2024-11-25T05:54:44,966 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6586672e-597e-41c1-14d9-2bcb3b602f07/hadoop.log.dir in system properties and HBase conf 2024-11-25T05:54:44,967 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6586672e-597e-41c1-14d9-2bcb3b602f07/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-25T05:54:44,968 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6586672e-597e-41c1-14d9-2bcb3b602f07/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-25T05:54:44,968 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-25T05:54:45,085 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-25T05:54:45,220 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T05:54:45,226 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6586672e-597e-41c1-14d9-2bcb3b602f07/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-25T05:54:45,226 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6586672e-597e-41c1-14d9-2bcb3b602f07/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-25T05:54:45,227 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6586672e-597e-41c1-14d9-2bcb3b602f07/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-25T05:54:45,228 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6586672e-597e-41c1-14d9-2bcb3b602f07/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-25T05:54:45,228 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6586672e-597e-41c1-14d9-2bcb3b602f07/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-25T05:54:45,229 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6586672e-597e-41c1-14d9-2bcb3b602f07/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-25T05:54:45,229 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6586672e-597e-41c1-14d9-2bcb3b602f07/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-25T05:54:45,230 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6586672e-597e-41c1-14d9-2bcb3b602f07/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-25T05:54:45,231 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6586672e-597e-41c1-14d9-2bcb3b602f07/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-25T05:54:45,231 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6586672e-597e-41c1-14d9-2bcb3b602f07/nfs.dump.dir in system properties and HBase conf 2024-11-25T05:54:45,232 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6586672e-597e-41c1-14d9-2bcb3b602f07/java.io.tmpdir in system properties and HBase conf 2024-11-25T05:54:45,232 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6586672e-597e-41c1-14d9-2bcb3b602f07/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-25T05:54:45,233 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6586672e-597e-41c1-14d9-2bcb3b602f07/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-25T05:54:45,233 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6586672e-597e-41c1-14d9-2bcb3b602f07/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-25T05:54:46,029 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-25T05:54:46,116 INFO [Time-limited test {}] log.Log(170): Logging initialized @2274ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-25T05:54:46,204 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T05:54:46,271 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T05:54:46,293 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T05:54:46,294 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T05:54:46,295 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-25T05:54:46,306 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T05:54:46,309 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6586672e-597e-41c1-14d9-2bcb3b602f07/hadoop.log.dir/,AVAILABLE} 2024-11-25T05:54:46,310 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T05:54:46,478 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6586672e-597e-41c1-14d9-2bcb3b602f07/java.io.tmpdir/jetty-localhost-39979-hadoop-hdfs-3_4_1-tests_jar-_-any-15642314378173223613/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-25T05:54:46,484 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:39979} 2024-11-25T05:54:46,485 INFO [Time-limited test {}] server.Server(415): Started @2643ms 2024-11-25T05:54:46,826 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T05:54:46,833 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T05:54:46,835 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T05:54:46,835 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T05:54:46,835 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-25T05:54:46,836 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6586672e-597e-41c1-14d9-2bcb3b602f07/hadoop.log.dir/,AVAILABLE} 2024-11-25T05:54:46,836 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T05:54:46,937 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6586672e-597e-41c1-14d9-2bcb3b602f07/java.io.tmpdir/jetty-localhost-38093-hadoop-hdfs-3_4_1-tests_jar-_-any-5054184549019373414/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T05:54:46,938 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:38093} 2024-11-25T05:54:46,938 INFO [Time-limited test {}] server.Server(415): Started @3096ms 2024-11-25T05:54:46,988 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-25T05:54:47,385 WARN [Thread-70 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6586672e-597e-41c1-14d9-2bcb3b602f07/cluster_8af6f6a5-ddee-228b-0d29-2ed39e53a991/data/data1/current/BP-1069327184-172.17.0.2-1732514085807/current, will proceed with Du for space computation calculation, 2024-11-25T05:54:47,385 WARN [Thread-71 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6586672e-597e-41c1-14d9-2bcb3b602f07/cluster_8af6f6a5-ddee-228b-0d29-2ed39e53a991/data/data2/current/BP-1069327184-172.17.0.2-1732514085807/current, will proceed with Du for space computation calculation, 2024-11-25T05:54:47,433 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-25T05:54:47,491 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x97372991b7cebd5c with lease ID 0xee82b25fd8d17d81: Processing first storage report for DS-eebbc315-faf7-4a85-b2ca-cbc66a0c0bab from datanode DatanodeRegistration(127.0.0.1:33269, datanodeUuid=722df91b-1e78-4820-8141-ffa7be2e3ce0, infoPort=42525, infoSecurePort=0, ipcPort=36767, storageInfo=lv=-57;cid=testClusterID;nsid=1731655827;c=1732514085807) 2024-11-25T05:54:47,493 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x97372991b7cebd5c with lease ID 0xee82b25fd8d17d81: from storage DS-eebbc315-faf7-4a85-b2ca-cbc66a0c0bab node DatanodeRegistration(127.0.0.1:33269, datanodeUuid=722df91b-1e78-4820-8141-ffa7be2e3ce0, infoPort=42525, infoSecurePort=0, ipcPort=36767, storageInfo=lv=-57;cid=testClusterID;nsid=1731655827;c=1732514085807), blocks: 0, hasStaleStorage: true, processing time: 3 msecs, invalidatedBlocks: 0 2024-11-25T05:54:47,494 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x97372991b7cebd5c with lease ID 0xee82b25fd8d17d81: Processing first storage report for DS-17447f67-1e04-4c42-b937-e9b09137668c from datanode DatanodeRegistration(127.0.0.1:33269, datanodeUuid=722df91b-1e78-4820-8141-ffa7be2e3ce0, infoPort=42525, infoSecurePort=0, ipcPort=36767, storageInfo=lv=-57;cid=testClusterID;nsid=1731655827;c=1732514085807) 2024-11-25T05:54:47,494 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x97372991b7cebd5c with lease ID 0xee82b25fd8d17d81: from storage DS-17447f67-1e04-4c42-b937-e9b09137668c node DatanodeRegistration(127.0.0.1:33269, datanodeUuid=722df91b-1e78-4820-8141-ffa7be2e3ce0, infoPort=42525, infoSecurePort=0, ipcPort=36767, storageInfo=lv=-57;cid=testClusterID;nsid=1731655827;c=1732514085807), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-25T05:54:47,549 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6586672e-597e-41c1-14d9-2bcb3b602f07 2024-11-25T05:54:47,619 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6586672e-597e-41c1-14d9-2bcb3b602f07/cluster_8af6f6a5-ddee-228b-0d29-2ed39e53a991/zookeeper_0, clientPort=59687, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6586672e-597e-41c1-14d9-2bcb3b602f07/cluster_8af6f6a5-ddee-228b-0d29-2ed39e53a991/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6586672e-597e-41c1-14d9-2bcb3b602f07/cluster_8af6f6a5-ddee-228b-0d29-2ed39e53a991/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-25T05:54:47,628 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59687 2024-11-25T05:54:47,644 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T05:54:47,649 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T05:54:47,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741825_1001 (size=7) 2024-11-25T05:54:48,277 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a with version=8 2024-11-25T05:54:48,277 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/hbase-staging 2024-11-25T05:54:48,357 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-25T05:54:48,560 INFO [Time-limited test {}] client.ConnectionUtils(128): master/8ef925b832e3:0 server-side Connection retries=45 2024-11-25T05:54:48,569 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T05:54:48,570 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T05:54:48,575 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T05:54:48,575 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T05:54:48,575 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T05:54:48,717 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-25T05:54:48,768 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-25T05:54:48,779 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-25T05:54:48,784 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T05:54:48,811 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 94165 (auto-detected) 2024-11-25T05:54:48,812 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-25T05:54:48,830 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41285 2024-11-25T05:54:48,849 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41285 connecting to ZooKeeper ensemble=127.0.0.1:59687 2024-11-25T05:54:48,873 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:412850x0, quorum=127.0.0.1:59687, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T05:54:48,876 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41285-0x10075725f200000 connected 2024-11-25T05:54:48,901 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T05:54:48,903 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T05:54:48,912 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41285-0x10075725f200000, quorum=127.0.0.1:59687, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T05:54:48,915 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a, hbase.cluster.distributed=false 2024-11-25T05:54:48,936 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41285-0x10075725f200000, quorum=127.0.0.1:59687, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T05:54:48,940 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41285 2024-11-25T05:54:48,940 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41285 2024-11-25T05:54:48,941 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41285 2024-11-25T05:54:48,942 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41285 2024-11-25T05:54:48,943 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41285 2024-11-25T05:54:49,045 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/8ef925b832e3:0 server-side Connection retries=45 2024-11-25T05:54:49,047 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T05:54:49,047 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T05:54:49,047 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T05:54:49,047 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T05:54:49,047 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T05:54:49,050 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T05:54:49,052 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T05:54:49,053 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35599 2024-11-25T05:54:49,055 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35599 connecting to ZooKeeper ensemble=127.0.0.1:59687 2024-11-25T05:54:49,056 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T05:54:49,059 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T05:54:49,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:355990x0, quorum=127.0.0.1:59687, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T05:54:49,065 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:355990x0, quorum=127.0.0.1:59687, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T05:54:49,066 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35599-0x10075725f200001 connected 2024-11-25T05:54:49,069 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T05:54:49,076 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T05:54:49,079 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35599-0x10075725f200001, quorum=127.0.0.1:59687, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T05:54:49,084 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35599-0x10075725f200001, quorum=127.0.0.1:59687, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T05:54:49,085 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35599 2024-11-25T05:54:49,085 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35599 2024-11-25T05:54:49,089 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35599 2024-11-25T05:54:49,091 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35599 2024-11-25T05:54:49,091 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35599 2024-11-25T05:54:49,105 DEBUG [M:0;8ef925b832e3:41285 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;8ef925b832e3:41285 2024-11-25T05:54:49,106 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/8ef925b832e3,41285,1732514088402 2024-11-25T05:54:49,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35599-0x10075725f200001, quorum=127.0.0.1:59687, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T05:54:49,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41285-0x10075725f200000, quorum=127.0.0.1:59687, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T05:54:49,113 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41285-0x10075725f200000, quorum=127.0.0.1:59687, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/8ef925b832e3,41285,1732514088402 2024-11-25T05:54:49,137 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35599-0x10075725f200001, quorum=127.0.0.1:59687, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T05:54:49,137 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41285-0x10075725f200000, quorum=127.0.0.1:59687, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T05:54:49,137 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35599-0x10075725f200001, quorum=127.0.0.1:59687, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T05:54:49,138 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41285-0x10075725f200000, quorum=127.0.0.1:59687, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-25T05:54:49,139 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/8ef925b832e3,41285,1732514088402 from backup master directory 2024-11-25T05:54:49,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35599-0x10075725f200001, quorum=127.0.0.1:59687, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T05:54:49,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41285-0x10075725f200000, quorum=127.0.0.1:59687, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/8ef925b832e3,41285,1732514088402 2024-11-25T05:54:49,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41285-0x10075725f200000, quorum=127.0.0.1:59687, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T05:54:49,142 WARN [master/8ef925b832e3:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T05:54:49,143 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=8ef925b832e3,41285,1732514088402 2024-11-25T05:54:49,145 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-25T05:54:49,146 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-25T05:54:49,198 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/hbase.id] with ID: 0e308606-44a7-4b56-9484-ef3e65c21c05 2024-11-25T05:54:49,198 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/.tmp/hbase.id 2024-11-25T05:54:49,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741826_1002 (size=42) 2024-11-25T05:54:49,615 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/.tmp/hbase.id]:[hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/hbase.id] 2024-11-25T05:54:49,667 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T05:54:49,673 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-25T05:54:49,693 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-25T05:54:49,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41285-0x10075725f200000, quorum=127.0.0.1:59687, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T05:54:49,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35599-0x10075725f200001, quorum=127.0.0.1:59687, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T05:54:49,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741827_1003 (size=196) 2024-11-25T05:54:50,135 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T05:54:50,138 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-25T05:54:50,153 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:50,156 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T05:54:50,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741828_1004 (size=1189) 2024-11-25T05:54:50,610 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/MasterData/data/master/store 2024-11-25T05:54:50,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741829_1005 (size=34) 2024-11-25T05:54:51,040 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-25T05:54:51,043 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T05:54:51,044 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T05:54:51,044 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T05:54:51,045 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T05:54:51,046 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T05:54:51,046 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T05:54:51,046 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T05:54:51,048 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732514091044Disabling compacts and flushes for region at 1732514091044Disabling writes for close at 1732514091046 (+2 ms)Writing region close event to WAL at 1732514091046Closed at 1732514091046 2024-11-25T05:54:51,050 WARN [master/8ef925b832e3:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/MasterData/data/master/store/.initializing 2024-11-25T05:54:51,050 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/MasterData/WALs/8ef925b832e3,41285,1732514088402 2024-11-25T05:54:51,058 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T05:54:51,073 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8ef925b832e3%2C41285%2C1732514088402, suffix=, logDir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/MasterData/WALs/8ef925b832e3,41285,1732514088402, archiveDir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/MasterData/oldWALs, maxLogs=10 2024-11-25T05:54:51,094 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/MasterData/WALs/8ef925b832e3,41285,1732514088402/8ef925b832e3%2C41285%2C1732514088402.1732514091077, exclude list is [], retry=0 2024-11-25T05:54:51,110 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33269,DS-eebbc315-faf7-4a85-b2ca-cbc66a0c0bab,DISK] 2024-11-25T05:54:51,113 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-25T05:54:51,144 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/MasterData/WALs/8ef925b832e3,41285,1732514088402/8ef925b832e3%2C41285%2C1732514088402.1732514091077 2024-11-25T05:54:51,145 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42525:42525)] 2024-11-25T05:54:51,146 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-25T05:54:51,146 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T05:54:51,149 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T05:54:51,150 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T05:54:51,185 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T05:54:51,207 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-25T05:54:51,210 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:54:51,213 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T05:54:51,213 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T05:54:51,216 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-25T05:54:51,217 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:54:51,218 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T05:54:51,218 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T05:54:51,221 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-25T05:54:51,221 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:54:51,222 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T05:54:51,222 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T05:54:51,225 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-25T05:54:51,225 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:54:51,226 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T05:54:51,227 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T05:54:51,230 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T05:54:51,231 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T05:54:51,236 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T05:54:51,236 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T05:54:51,241 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T05:54:51,246 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T05:54:51,251 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T05:54:51,252 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62969132, jitterRate=-0.061686813831329346}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T05:54:51,259 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732514091162Initializing all the Stores at 1732514091164 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732514091165 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732514091165Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732514091166 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732514091166Cleaning up temporary data from old regions at 1732514091236 (+70 ms)Region opened successfully at 1732514091259 (+23 ms) 2024-11-25T05:54:51,260 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-25T05:54:51,288 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1695c839, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8ef925b832e3/172.17.0.2:0 2024-11-25T05:54:51,315 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-25T05:54:51,324 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-25T05:54:51,325 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-25T05:54:51,327 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T05:54:51,328 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-25T05:54:51,332 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-25T05:54:51,332 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-25T05:54:51,356 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-25T05:54:51,363 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41285-0x10075725f200000, quorum=127.0.0.1:59687, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-25T05:54:51,365 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-25T05:54:51,367 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-25T05:54:51,368 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41285-0x10075725f200000, quorum=127.0.0.1:59687, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-25T05:54:51,370 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-25T05:54:51,372 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-25T05:54:51,375 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41285-0x10075725f200000, quorum=127.0.0.1:59687, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-25T05:54:51,376 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-25T05:54:51,378 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41285-0x10075725f200000, quorum=127.0.0.1:59687, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-25T05:54:51,379 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-25T05:54:51,393 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41285-0x10075725f200000, quorum=127.0.0.1:59687, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-25T05:54:51,394 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-25T05:54:51,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35599-0x10075725f200001, quorum=127.0.0.1:59687, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T05:54:51,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41285-0x10075725f200000, quorum=127.0.0.1:59687, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T05:54:51,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35599-0x10075725f200001, quorum=127.0.0.1:59687, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T05:54:51,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41285-0x10075725f200000, quorum=127.0.0.1:59687, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T05:54:51,400 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=8ef925b832e3,41285,1732514088402, sessionid=0x10075725f200000, setting cluster-up flag (Was=false) 2024-11-25T05:54:51,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35599-0x10075725f200001, quorum=127.0.0.1:59687, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T05:54:51,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41285-0x10075725f200000, quorum=127.0.0.1:59687, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T05:54:51,414 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-25T05:54:51,416 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=8ef925b832e3,41285,1732514088402 2024-11-25T05:54:51,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41285-0x10075725f200000, quorum=127.0.0.1:59687, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T05:54:51,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35599-0x10075725f200001, quorum=127.0.0.1:59687, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T05:54:51,424 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-25T05:54:51,426 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=8ef925b832e3,41285,1732514088402 2024-11-25T05:54:51,431 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-25T05:54:51,498 INFO [RS:0;8ef925b832e3:35599 {}] regionserver.HRegionServer(746): ClusterId : 0e308606-44a7-4b56-9484-ef3e65c21c05 2024-11-25T05:54:51,500 DEBUG [RS:0;8ef925b832e3:35599 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T05:54:51,500 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T05:54:51,504 DEBUG [RS:0;8ef925b832e3:35599 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T05:54:51,504 DEBUG [RS:0;8ef925b832e3:35599 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T05:54:51,507 DEBUG [RS:0;8ef925b832e3:35599 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T05:54:51,507 DEBUG [RS:0;8ef925b832e3:35599 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4554fe5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8ef925b832e3/172.17.0.2:0 2024-11-25T05:54:51,510 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-25T05:54:51,517 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-25T05:54:51,521 DEBUG [RS:0;8ef925b832e3:35599 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;8ef925b832e3:35599 2024-11-25T05:54:51,524 INFO [RS:0;8ef925b832e3:35599 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T05:54:51,524 INFO [RS:0;8ef925b832e3:35599 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T05:54:51,525 DEBUG [RS:0;8ef925b832e3:35599 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T05:54:51,522 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 8ef925b832e3,41285,1732514088402 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-25T05:54:51,527 INFO [RS:0;8ef925b832e3:35599 {}] regionserver.HRegionServer(2659): reportForDuty to master=8ef925b832e3,41285,1732514088402 with port=35599, startcode=1732514089005 2024-11-25T05:54:51,530 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/8ef925b832e3:0, corePoolSize=5, maxPoolSize=5 2024-11-25T05:54:51,530 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/8ef925b832e3:0, corePoolSize=5, maxPoolSize=5 2024-11-25T05:54:51,531 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/8ef925b832e3:0, corePoolSize=5, maxPoolSize=5 2024-11-25T05:54:51,531 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/8ef925b832e3:0, corePoolSize=5, maxPoolSize=5 2024-11-25T05:54:51,531 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/8ef925b832e3:0, corePoolSize=10, maxPoolSize=10 2024-11-25T05:54:51,531 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/8ef925b832e3:0, corePoolSize=1, maxPoolSize=1 2024-11-25T05:54:51,532 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/8ef925b832e3:0, corePoolSize=2, maxPoolSize=2 2024-11-25T05:54:51,532 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/8ef925b832e3:0, corePoolSize=1, maxPoolSize=1 2024-11-25T05:54:51,533 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732514121533 2024-11-25T05:54:51,535 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-25T05:54:51,536 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-25T05:54:51,537 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T05:54:51,538 DEBUG [RS:0;8ef925b832e3:35599 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T05:54:51,538 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-25T05:54:51,540 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-25T05:54:51,540 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-25T05:54:51,540 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-25T05:54:51,541 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-25T05:54:51,541 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T05:54:51,544 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:54:51,545 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-25T05:54:51,545 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-25T05:54:51,546 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-25T05:54:51,547 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-25T05:54:51,551 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-25T05:54:51,551 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-25T05:54:51,557 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/8ef925b832e3:0:becomeActiveMaster-HFileCleaner.large.0-1732514091553,5,FailOnTimeoutGroup] 2024-11-25T05:54:51,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741831_1007 (size=1321) 2024-11-25T05:54:51,560 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/8ef925b832e3:0:becomeActiveMaster-HFileCleaner.small.0-1732514091558,5,FailOnTimeoutGroup] 2024-11-25T05:54:51,561 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T05:54:51,561 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-25T05:54:51,562 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-25T05:54:51,562 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-25T05:54:51,562 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-25T05:54:51,563 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a 2024-11-25T05:54:51,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741832_1008 (size=32) 2024-11-25T05:54:51,580 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38147, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T05:54:51,585 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41285 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 8ef925b832e3,35599,1732514089005 2024-11-25T05:54:51,587 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41285 {}] master.ServerManager(517): Registering regionserver=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:51,601 DEBUG [RS:0;8ef925b832e3:35599 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a 2024-11-25T05:54:51,602 DEBUG [RS:0;8ef925b832e3:35599 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42949 2024-11-25T05:54:51,602 DEBUG [RS:0;8ef925b832e3:35599 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T05:54:51,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41285-0x10075725f200000, quorum=127.0.0.1:59687, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T05:54:51,607 DEBUG [RS:0;8ef925b832e3:35599 {}] zookeeper.ZKUtil(111): regionserver:35599-0x10075725f200001, quorum=127.0.0.1:59687, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/8ef925b832e3,35599,1732514089005 2024-11-25T05:54:51,607 WARN [RS:0;8ef925b832e3:35599 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T05:54:51,607 INFO [RS:0;8ef925b832e3:35599 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T05:54:51,607 DEBUG [RS:0;8ef925b832e3:35599 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/WALs/8ef925b832e3,35599,1732514089005 2024-11-25T05:54:51,609 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [8ef925b832e3,35599,1732514089005] 2024-11-25T05:54:51,632 INFO [RS:0;8ef925b832e3:35599 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T05:54:51,649 INFO [RS:0;8ef925b832e3:35599 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T05:54:51,654 INFO [RS:0;8ef925b832e3:35599 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T05:54:51,654 INFO [RS:0;8ef925b832e3:35599 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T05:54:51,655 INFO [RS:0;8ef925b832e3:35599 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T05:54:51,661 INFO [RS:0;8ef925b832e3:35599 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T05:54:51,662 INFO [RS:0;8ef925b832e3:35599 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T05:54:51,662 DEBUG [RS:0;8ef925b832e3:35599 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/8ef925b832e3:0, corePoolSize=1, maxPoolSize=1 2024-11-25T05:54:51,663 DEBUG [RS:0;8ef925b832e3:35599 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/8ef925b832e3:0, corePoolSize=1, maxPoolSize=1 2024-11-25T05:54:51,663 DEBUG [RS:0;8ef925b832e3:35599 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/8ef925b832e3:0, corePoolSize=1, maxPoolSize=1 2024-11-25T05:54:51,663 DEBUG [RS:0;8ef925b832e3:35599 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/8ef925b832e3:0, corePoolSize=1, maxPoolSize=1 2024-11-25T05:54:51,663 DEBUG [RS:0;8ef925b832e3:35599 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/8ef925b832e3:0, corePoolSize=1, maxPoolSize=1 2024-11-25T05:54:51,663 DEBUG [RS:0;8ef925b832e3:35599 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/8ef925b832e3:0, corePoolSize=2, maxPoolSize=2 2024-11-25T05:54:51,663 DEBUG [RS:0;8ef925b832e3:35599 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0, corePoolSize=1, maxPoolSize=1 2024-11-25T05:54:51,663 DEBUG [RS:0;8ef925b832e3:35599 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/8ef925b832e3:0, corePoolSize=1, maxPoolSize=1 2024-11-25T05:54:51,664 DEBUG [RS:0;8ef925b832e3:35599 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/8ef925b832e3:0, corePoolSize=1, maxPoolSize=1 2024-11-25T05:54:51,664 DEBUG [RS:0;8ef925b832e3:35599 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/8ef925b832e3:0, corePoolSize=1, maxPoolSize=1 2024-11-25T05:54:51,664 DEBUG [RS:0;8ef925b832e3:35599 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/8ef925b832e3:0, corePoolSize=1, maxPoolSize=1 2024-11-25T05:54:51,664 DEBUG [RS:0;8ef925b832e3:35599 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/8ef925b832e3:0, corePoolSize=1, maxPoolSize=1 2024-11-25T05:54:51,664 DEBUG [RS:0;8ef925b832e3:35599 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/8ef925b832e3:0, corePoolSize=3, maxPoolSize=3 2024-11-25T05:54:51,664 DEBUG [RS:0;8ef925b832e3:35599 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0, corePoolSize=3, maxPoolSize=3 2024-11-25T05:54:51,666 INFO [RS:0;8ef925b832e3:35599 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T05:54:51,666 INFO [RS:0;8ef925b832e3:35599 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T05:54:51,666 INFO [RS:0;8ef925b832e3:35599 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T05:54:51,666 INFO [RS:0;8ef925b832e3:35599 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T05:54:51,666 INFO [RS:0;8ef925b832e3:35599 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T05:54:51,667 INFO [RS:0;8ef925b832e3:35599 {}] hbase.ChoreService(168): Chore ScheduledChore name=8ef925b832e3,35599,1732514089005-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T05:54:51,685 INFO [RS:0;8ef925b832e3:35599 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T05:54:51,687 INFO [RS:0;8ef925b832e3:35599 {}] hbase.ChoreService(168): Chore ScheduledChore name=8ef925b832e3,35599,1732514089005-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T05:54:51,687 INFO [RS:0;8ef925b832e3:35599 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T05:54:51,687 INFO [RS:0;8ef925b832e3:35599 {}] regionserver.Replication(171): 8ef925b832e3,35599,1732514089005 started 2024-11-25T05:54:51,704 INFO [RS:0;8ef925b832e3:35599 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T05:54:51,704 INFO [RS:0;8ef925b832e3:35599 {}] regionserver.HRegionServer(1482): Serving as 8ef925b832e3,35599,1732514089005, RpcServer on 8ef925b832e3/172.17.0.2:35599, sessionid=0x10075725f200001 2024-11-25T05:54:51,705 DEBUG [RS:0;8ef925b832e3:35599 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T05:54:51,705 DEBUG [RS:0;8ef925b832e3:35599 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 8ef925b832e3,35599,1732514089005 2024-11-25T05:54:51,705 DEBUG [RS:0;8ef925b832e3:35599 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8ef925b832e3,35599,1732514089005' 2024-11-25T05:54:51,705 DEBUG [RS:0;8ef925b832e3:35599 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T05:54:51,706 DEBUG [RS:0;8ef925b832e3:35599 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T05:54:51,707 DEBUG [RS:0;8ef925b832e3:35599 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T05:54:51,707 DEBUG [RS:0;8ef925b832e3:35599 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T05:54:51,707 DEBUG [RS:0;8ef925b832e3:35599 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 8ef925b832e3,35599,1732514089005 2024-11-25T05:54:51,707 DEBUG [RS:0;8ef925b832e3:35599 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8ef925b832e3,35599,1732514089005' 2024-11-25T05:54:51,708 DEBUG [RS:0;8ef925b832e3:35599 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T05:54:51,708 DEBUG [RS:0;8ef925b832e3:35599 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T05:54:51,709 DEBUG [RS:0;8ef925b832e3:35599 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T05:54:51,709 INFO [RS:0;8ef925b832e3:35599 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T05:54:51,709 INFO [RS:0;8ef925b832e3:35599 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T05:54:51,818 INFO [RS:0;8ef925b832e3:35599 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T05:54:51,822 INFO [RS:0;8ef925b832e3:35599 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8ef925b832e3%2C35599%2C1732514089005, suffix=, logDir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/WALs/8ef925b832e3,35599,1732514089005, archiveDir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/oldWALs, maxLogs=32 2024-11-25T05:54:51,836 DEBUG [RS:0;8ef925b832e3:35599 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/WALs/8ef925b832e3,35599,1732514089005/8ef925b832e3%2C35599%2C1732514089005.1732514091824, exclude list is [], retry=0 2024-11-25T05:54:51,840 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33269,DS-eebbc315-faf7-4a85-b2ca-cbc66a0c0bab,DISK] 2024-11-25T05:54:51,845 INFO [RS:0;8ef925b832e3:35599 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/WALs/8ef925b832e3,35599,1732514089005/8ef925b832e3%2C35599%2C1732514089005.1732514091824 2024-11-25T05:54:51,846 DEBUG [RS:0;8ef925b832e3:35599 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42525:42525)] 2024-11-25T05:54:51,975 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T05:54:51,978 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-25T05:54:51,980 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T05:54:51,980 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:54:51,981 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T05:54:51,982 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-25T05:54:51,984 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T05:54:51,984 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:54:51,985 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T05:54:51,985 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-25T05:54:51,987 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T05:54:51,987 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:54:51,988 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T05:54:51,988 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-25T05:54:51,990 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T05:54:51,991 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:54:51,992 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T05:54:51,992 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T05:54:51,993 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/hbase/meta/1588230740 2024-11-25T05:54:51,994 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/hbase/meta/1588230740 2024-11-25T05:54:51,997 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T05:54:51,997 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T05:54:51,998 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T05:54:52,000 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T05:54:52,004 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T05:54:52,005 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60872576, jitterRate=-0.09292793273925781}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T05:54:52,007 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732514091975Initializing all the Stores at 1732514091977 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732514091977Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732514091977Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732514091977Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732514091978 (+1 ms)Cleaning up temporary data from old regions at 1732514091997 (+19 ms)Region opened successfully at 1732514092007 (+10 ms) 2024-11-25T05:54:52,007 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T05:54:52,007 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T05:54:52,007 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T05:54:52,007 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T05:54:52,007 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T05:54:52,009 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T05:54:52,009 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732514092007Disabling compacts and flushes for region at 1732514092007Disabling writes for close at 1732514092007Writing region close event to WAL at 1732514092008 (+1 ms)Closed at 1732514092009 (+1 ms) 2024-11-25T05:54:52,012 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T05:54:52,012 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-25T05:54:52,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-25T05:54:52,026 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T05:54:52,029 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-25T05:54:52,185 DEBUG [8ef925b832e3:41285 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-25T05:54:52,196 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:52,202 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8ef925b832e3,35599,1732514089005, state=OPENING 2024-11-25T05:54:52,206 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-25T05:54:52,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35599-0x10075725f200001, quorum=127.0.0.1:59687, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T05:54:52,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41285-0x10075725f200000, quorum=127.0.0.1:59687, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T05:54:52,208 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T05:54:52,208 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T05:54:52,210 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T05:54:52,211 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=8ef925b832e3,35599,1732514089005}] 2024-11-25T05:54:52,387 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T05:54:52,390 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39549, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T05:54:52,400 INFO [RS_OPEN_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-25T05:54:52,401 INFO [RS_OPEN_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T05:54:52,401 INFO [RS_OPEN_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-25T05:54:52,404 INFO [RS_OPEN_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8ef925b832e3%2C35599%2C1732514089005.meta, suffix=.meta, logDir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/WALs/8ef925b832e3,35599,1732514089005, archiveDir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/oldWALs, maxLogs=32 2024-11-25T05:54:52,418 DEBUG [RS_OPEN_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/WALs/8ef925b832e3,35599,1732514089005/8ef925b832e3%2C35599%2C1732514089005.meta.1732514092406.meta, exclude list is [], retry=0 2024-11-25T05:54:52,421 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33269,DS-eebbc315-faf7-4a85-b2ca-cbc66a0c0bab,DISK] 2024-11-25T05:54:52,424 INFO [RS_OPEN_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/WALs/8ef925b832e3,35599,1732514089005/8ef925b832e3%2C35599%2C1732514089005.meta.1732514092406.meta 2024-11-25T05:54:52,425 DEBUG [RS_OPEN_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42525:42525)] 2024-11-25T05:54:52,425 DEBUG [RS_OPEN_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-25T05:54:52,426 DEBUG [RS_OPEN_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-25T05:54:52,429 DEBUG [RS_OPEN_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-25T05:54:52,433 INFO [RS_OPEN_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-25T05:54:52,436 DEBUG [RS_OPEN_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-25T05:54:52,437 DEBUG [RS_OPEN_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T05:54:52,437 DEBUG [RS_OPEN_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-25T05:54:52,437 DEBUG [RS_OPEN_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-25T05:54:52,440 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-25T05:54:52,442 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T05:54:52,442 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:54:52,443 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T05:54:52,443 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-25T05:54:52,445 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T05:54:52,445 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:54:52,446 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T05:54:52,446 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-25T05:54:52,448 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T05:54:52,448 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:54:52,449 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T05:54:52,449 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-25T05:54:52,451 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T05:54:52,451 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:54:52,452 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T05:54:52,452 DEBUG [RS_OPEN_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T05:54:52,453 DEBUG [RS_OPEN_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/hbase/meta/1588230740 2024-11-25T05:54:52,456 DEBUG [RS_OPEN_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/hbase/meta/1588230740 2024-11-25T05:54:52,459 DEBUG [RS_OPEN_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T05:54:52,459 DEBUG [RS_OPEN_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T05:54:52,460 DEBUG [RS_OPEN_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T05:54:52,462 DEBUG [RS_OPEN_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T05:54:52,464 INFO [RS_OPEN_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74021539, jitterRate=0.10300688445568085}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T05:54:52,465 DEBUG [RS_OPEN_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-25T05:54:52,466 DEBUG [RS_OPEN_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732514092438Writing region info on filesystem at 1732514092438Initializing all the Stores at 1732514092440 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732514092440Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732514092440Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732514092440Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732514092440Cleaning up temporary data from old regions at 1732514092459 (+19 ms)Running coprocessor post-open hooks at 1732514092465 (+6 ms)Region opened successfully at 1732514092466 (+1 ms) 2024-11-25T05:54:52,472 INFO [RS_OPEN_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732514092379 2024-11-25T05:54:52,483 DEBUG [RS_OPEN_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-25T05:54:52,484 INFO [RS_OPEN_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-25T05:54:52,485 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:52,488 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8ef925b832e3,35599,1732514089005, state=OPEN 2024-11-25T05:54:52,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41285-0x10075725f200000, quorum=127.0.0.1:59687, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T05:54:52,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35599-0x10075725f200001, quorum=127.0.0.1:59687, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T05:54:52,491 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T05:54:52,491 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T05:54:52,492 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:52,499 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-25T05:54:52,499 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=8ef925b832e3,35599,1732514089005 in 281 msec 2024-11-25T05:54:52,507 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-25T05:54:52,507 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 483 msec 2024-11-25T05:54:52,509 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T05:54:52,509 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-25T05:54:52,531 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:54:52,532 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:54:52,556 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:54:52,559 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49233, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:54:52,580 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1180 sec 2024-11-25T05:54:52,581 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732514092581, completionTime=-1 2024-11-25T05:54:52,583 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-25T05:54:52,583 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T05:54:52,609 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-25T05:54:52,609 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732514152609 2024-11-25T05:54:52,609 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732514212609 2024-11-25T05:54:52,609 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec 2024-11-25T05:54:52,611 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8ef925b832e3,41285,1732514088402-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T05:54:52,612 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8ef925b832e3,41285,1732514088402-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T05:54:52,612 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8ef925b832e3,41285,1732514088402-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T05:54:52,613 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-8ef925b832e3:41285, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T05:54:52,613 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-25T05:54:52,614 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-25T05:54:52,619 DEBUG [master/8ef925b832e3:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-25T05:54:52,640 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.497sec 2024-11-25T05:54:52,641 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-25T05:54:52,642 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-25T05:54:52,643 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-25T05:54:52,643 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-25T05:54:52,643 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-25T05:54:52,644 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8ef925b832e3,41285,1732514088402-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T05:54:52,644 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8ef925b832e3,41285,1732514088402-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-25T05:54:52,652 DEBUG [master/8ef925b832e3:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-25T05:54:52,653 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-25T05:54:52,653 INFO [master/8ef925b832e3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8ef925b832e3,41285,1732514088402-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T05:54:52,713 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bca31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:54:52,715 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-25T05:54:52,715 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-25T05:54:52,717 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:54:52,720 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:54:52,726 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:54:52,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:54:52,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:54:52,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@109c0bbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:54:52,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:54:52,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:54:52,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:54:52,735 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37434, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:54:52,738 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@512c5941, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:54:52,739 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:54:52,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:54:52,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:54:52,748 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47332, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:54:52,781 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=8ef925b832e3,41285,1732514088402 2024-11-25T05:54:52,794 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=210, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=88, ProcessCount=11, AvailableMemoryMB=7722 2024-11-25T05:54:52,802 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T05:54:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 8ef925b832e3,41285,1732514088402 2024-11-25T05:54:52,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@64e9cfde 2024-11-25T05:54:52,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T05:54:52,819 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37444, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T05:54:52,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T05:54:52,825 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T05:54:52,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-25T05:54:52,836 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T05:54:52,839 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-25T05:54:52,839 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:54:52,842 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T05:54:52,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T05:54:52,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741835_1011 (size=1061) 2024-11-25T05:54:52,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T05:54:53,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T05:54:53,258 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a 2024-11-25T05:54:53,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741836_1012 (size=53) 2024-11-25T05:54:53,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T05:54:53,670 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T05:54:53,670 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing cf8ce3f4f3ad5ac4b5297b0ca903cd34, disabling compactions & flushes 2024-11-25T05:54:53,670 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:53,670 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:53,671 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. after waiting 0 ms 2024-11-25T05:54:53,671 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:53,671 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:53,671 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: Waiting for close lock at 1732514093670Disabling compacts and flushes for region at 1732514093670Disabling writes for close at 1732514093671 (+1 ms)Writing region close event to WAL at 1732514093671Closed at 1732514093671 2024-11-25T05:54:53,674 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T05:54:53,682 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732514093674"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732514093674"}]},"ts":"1732514093674"} 2024-11-25T05:54:53,687 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T05:54:53,689 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T05:54:53,691 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732514093689"}]},"ts":"1732514093689"} 2024-11-25T05:54:53,696 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-25T05:54:53,698 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf8ce3f4f3ad5ac4b5297b0ca903cd34, ASSIGN}] 2024-11-25T05:54:53,701 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf8ce3f4f3ad5ac4b5297b0ca903cd34, ASSIGN 2024-11-25T05:54:53,703 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf8ce3f4f3ad5ac4b5297b0ca903cd34, ASSIGN; state=OFFLINE, location=8ef925b832e3,35599,1732514089005; forceNewPlan=false, retain=false 2024-11-25T05:54:53,855 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=cf8ce3f4f3ad5ac4b5297b0ca903cd34, regionState=OPENING, regionLocation=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:53,863 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf8ce3f4f3ad5ac4b5297b0ca903cd34, ASSIGN because future has completed 2024-11-25T05:54:53,864 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005}] 2024-11-25T05:54:53,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T05:54:54,026 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:54,026 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} 2024-11-25T05:54:54,027 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:54:54,027 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T05:54:54,027 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:54:54,027 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:54:54,029 INFO [StoreOpener-cf8ce3f4f3ad5ac4b5297b0ca903cd34-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:54:54,034 INFO [StoreOpener-cf8ce3f4f3ad5ac4b5297b0ca903cd34-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T05:54:54,035 INFO [StoreOpener-cf8ce3f4f3ad5ac4b5297b0ca903cd34-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf8ce3f4f3ad5ac4b5297b0ca903cd34 columnFamilyName A 2024-11-25T05:54:54,035 DEBUG [StoreOpener-cf8ce3f4f3ad5ac4b5297b0ca903cd34-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:54:54,037 INFO [StoreOpener-cf8ce3f4f3ad5ac4b5297b0ca903cd34-1 {}] regionserver.HStore(327): Store=cf8ce3f4f3ad5ac4b5297b0ca903cd34/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T05:54:54,037 INFO [StoreOpener-cf8ce3f4f3ad5ac4b5297b0ca903cd34-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:54:54,040 INFO [StoreOpener-cf8ce3f4f3ad5ac4b5297b0ca903cd34-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T05:54:54,040 INFO [StoreOpener-cf8ce3f4f3ad5ac4b5297b0ca903cd34-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf8ce3f4f3ad5ac4b5297b0ca903cd34 columnFamilyName B 2024-11-25T05:54:54,040 DEBUG [StoreOpener-cf8ce3f4f3ad5ac4b5297b0ca903cd34-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:54:54,041 INFO [StoreOpener-cf8ce3f4f3ad5ac4b5297b0ca903cd34-1 {}] regionserver.HStore(327): Store=cf8ce3f4f3ad5ac4b5297b0ca903cd34/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T05:54:54,042 INFO [StoreOpener-cf8ce3f4f3ad5ac4b5297b0ca903cd34-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:54:54,044 INFO [StoreOpener-cf8ce3f4f3ad5ac4b5297b0ca903cd34-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T05:54:54,044 INFO [StoreOpener-cf8ce3f4f3ad5ac4b5297b0ca903cd34-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf8ce3f4f3ad5ac4b5297b0ca903cd34 columnFamilyName C 2024-11-25T05:54:54,045 DEBUG [StoreOpener-cf8ce3f4f3ad5ac4b5297b0ca903cd34-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:54:54,045 INFO [StoreOpener-cf8ce3f4f3ad5ac4b5297b0ca903cd34-1 {}] regionserver.HStore(327): Store=cf8ce3f4f3ad5ac4b5297b0ca903cd34/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T05:54:54,046 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:54,046 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:54:54,048 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:54:54,048 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:54:54,050 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:54:54,050 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:54:54,051 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T05:54:54,054 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:54:54,057 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T05:54:54,058 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened cf8ce3f4f3ad5ac4b5297b0ca903cd34; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66826538, jitterRate=-0.004206985235214233}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T05:54:54,058 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:54:54,059 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: Running coprocessor pre-open hook at 1732514094027Writing region info on filesystem at 1732514094027Initializing all the Stores at 1732514094029 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732514094029Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732514094029Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732514094029Cleaning up temporary data from old regions at 1732514094050 (+21 ms)Running coprocessor post-open hooks at 1732514094058 (+8 ms)Region opened successfully at 1732514094059 (+1 ms) 2024-11-25T05:54:54,061 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., pid=6, masterSystemTime=1732514094020 2024-11-25T05:54:54,065 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:54,065 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:54,066 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=cf8ce3f4f3ad5ac4b5297b0ca903cd34, regionState=OPEN, openSeqNum=2, regionLocation=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:54,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 because future has completed 2024-11-25T05:54:54,075 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-25T05:54:54,075 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 in 207 msec 2024-11-25T05:54:54,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-25T05:54:54,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf8ce3f4f3ad5ac4b5297b0ca903cd34, ASSIGN in 377 msec 2024-11-25T05:54:54,080 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T05:54:54,081 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732514094081"}]},"ts":"1732514094081"} 2024-11-25T05:54:54,084 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-25T05:54:54,086 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T05:54:54,089 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2580 sec 2024-11-25T05:54:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T05:54:54,997 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:54:55,008 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77c32821, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:54:55,008 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:54:55,008 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:54:55,010 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:54:55,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:54:55,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:54:55,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48a04f71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:54:55,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:54:55,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:54:55,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:54:55,013 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37462, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:54:55,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@435bed7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:54:55,015 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:54:55,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:54:55,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:54:55,019 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47336, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:54:55,022 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T05:54:55,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 8ef925b832e3,41285,1732514088402 2024-11-25T05:54:55,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6bf45604 2024-11-25T05:54:55,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T05:54:55,029 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37468, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T05:54:55,034 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bd90ee6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:54:55,034 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:54:55,034 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:54:55,035 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:54:55,036 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:54:55,036 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:54:55,036 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b3d2fd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:54:55,036 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:54:55,036 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:54:55,037 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:54:55,038 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37492, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:54:55,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c860641, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:54:55,040 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@577d2795, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:54:55,040 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:54:55,041 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:54:55,042 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:54:55,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:54:55,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:54:55,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@787f40de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:54:55,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:54:55,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:54:55,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:54:55,045 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37502, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:54:55,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@740f65e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:54:55,047 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bba02c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:54:55,047 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:54:55,047 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:54:55,048 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:54:55,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:54:55,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:54:55,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69b21050, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:54:55,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:54:55,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:54:55,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:54:55,052 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37512, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:54:55,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79382fcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:54:55,053 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@113d617f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:54:55,053 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:54:55,054 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:54:55,055 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:54:55,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:54:55,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:54:55,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45101745, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:54:55,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:54:55,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:54:55,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:54:55,058 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37534, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:54:55,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d0d5e57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:54:55,060 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2754d9b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:54:55,060 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:54:55,061 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:54:55,061 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:54:55,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:54:55,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:54:55,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63dc2eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:54:55,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:54:55,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:54:55,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:54:55,064 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37552, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:54:55,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@332ec47f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:54:55,066 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c3ddbdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:54:55,066 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:54:55,066 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:54:55,068 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:54:55,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:54:55,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:54:55,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d03237, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:54:55,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:54:55,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:54:55,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:54:55,070 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37572, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:54:55,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cd14d56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:54:55,072 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a48b566, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:54:55,072 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:54:55,072 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:54:55,073 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:54:55,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:54:55,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:54:55,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e9550bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:54:55,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:54:55,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:54:55,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:54:55,076 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37592, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:54:55,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b38edab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:54:55,078 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3111333, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:54:55,078 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:54:55,079 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:54:55,079 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:54:55,080 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:54:55,080 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:54:55,080 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1020fb6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:54:55,080 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:54:55,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:54:55,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:54:55,082 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37610, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:54:55,084 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57865cab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:54:55,084 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a046d27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:54:55,085 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:54:55,085 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:54:55,086 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:54:55,086 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:54:55,086 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:54:55,087 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37a0271, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:54:55,087 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:54:55,087 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:54:55,087 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:54:55,088 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37640, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:54:55,089 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9402b31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:54:55,092 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:54:55,092 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:54:55,092 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:54:55,092 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:54:55,093 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:54:55,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:54:55,095 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:54:55,095 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:54:55,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:54:55,095 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:54:55,095 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:54:55,095 DEBUG [Thread-147 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:54:55,095 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:54:55,095 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:54:55,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:54:55,096 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:54:55,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:54:55,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:54:55,097 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47344, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:54:55,097 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47342, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:54:55,098 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47340, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:54:55,098 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47362, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:54:55,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:54:55,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:54:55,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:54:55,099 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47366, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:54:55,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:54:55,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:54:55,101 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47380, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:54:55,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:54:55,102 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47386, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:54:55,103 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47400, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:54:55,104 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:54:55,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2] 2024-11-25T05:54:55,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2] 2024-11-25T05:54:55,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2] 2024-11-25T05:54:55,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2] 2024-11-25T05:54:55,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2] 2024-11-25T05:54:55,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:54:55,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:54:55,112 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:54:55,114 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:54:55,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2] 2024-11-25T05:54:55,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2] 2024-11-25T05:54:55,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2] 2024-11-25T05:54:55,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2] 2024-11-25T05:54:55,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-25T05:54:55,131 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:54:55,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T05:54:55,134 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:54:55,136 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:54:55,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:54:55,184 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-25T05:54:55,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=A 2024-11-25T05:54:55,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:54:55,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=B 2024-11-25T05:54:55,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:54:55,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=C 2024-11-25T05:54:55,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:54:55,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T05:54:55,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/3246d71eaa544a84af4849a86ce4a65c is 50, key is test_row_0/A:col10/1732514095123/Put/seqid=0 2024-11-25T05:54:55,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T05:54:55,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:55,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:54:55,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:55,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:55,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:55,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:55,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741837_1013 (size=16681) 2024-11-25T05:54:55,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/3246d71eaa544a84af4849a86ce4a65c 2024-11-25T05:54:55,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732514095280, max timestamp=1732514095350Num uniques 859; 2024-11-25T05:54:55,369 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T05:54:55,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732514095280, max timestamp=1732514095351Num uniques 859; 2024-11-25T05:54:55,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T05:54:55,380 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732514095280, max timestamp=1732514095350Num uniques 859; 2024-11-25T05:54:55,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T05:54:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T05:54:55,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514155444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:55,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:55,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514155449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:55,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:55,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514155450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:55,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514155455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:55,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:55,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514155460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:55,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:55,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:55,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:55,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:55,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:55,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:55,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:55,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:55,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:55,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:55,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:55,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:55,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T05:54:55,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:55,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:54:55,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:55,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:55,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/6d6bdc853a7d487ea551c72ca4d240ae is 50, key is test_row_0/B:col10/1732514095123/Put/seqid=0 2024-11-25T05:54:55,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:55,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:55,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:55,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:55,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:55,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741838_1014 (size=12001) 2024-11-25T05:54:55,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/6d6bdc853a7d487ea551c72ca4d240ae 2024-11-25T05:54:55,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:55,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:55,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514155614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:55,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:55,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:55,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/7abcf038322249a0b81f5e3fca326f2a is 50, key is test_row_0/C:col10/1732514095123/Put/seqid=0 2024-11-25T05:54:55,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:55,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514155618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:55,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:55,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:55,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514155614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:55,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:55,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:55,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:55,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514155619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:55,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:55,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:55,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:55,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514155624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:55,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:55,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:55,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:55,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741839_1015 (size=12001) 2024-11-25T05:54:55,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/7abcf038322249a0b81f5e3fca326f2a 2024-11-25T05:54:55,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/3246d71eaa544a84af4849a86ce4a65c as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/3246d71eaa544a84af4849a86ce4a65c 2024-11-25T05:54:55,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T05:54:55,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:55,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:54:55,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:55,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:55,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:55,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/3246d71eaa544a84af4849a86ce4a65c, entries=250, sequenceid=11, filesize=16.3 K 2024-11-25T05:54:55,701 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-25T05:54:55,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/6d6bdc853a7d487ea551c72ca4d240ae as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/6d6bdc853a7d487ea551c72ca4d240ae 2024-11-25T05:54:55,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/6d6bdc853a7d487ea551c72ca4d240ae, entries=150, sequenceid=11, filesize=11.7 K 2024-11-25T05:54:55,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/7abcf038322249a0b81f5e3fca326f2a as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/7abcf038322249a0b81f5e3fca326f2a 2024-11-25T05:54:55,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/7abcf038322249a0b81f5e3fca326f2a, entries=150, sequenceid=11, filesize=11.7 K 2024-11-25T05:54:55,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.42 KB/139690 for cf8ce3f4f3ad5ac4b5297b0ca903cd34 in 573ms, sequenceid=11, compaction requested=false 2024-11-25T05:54:55,761 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-25T05:54:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T05:54:55,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:54:55,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T05:54:55,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:55,840 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34 3/3 column families, dataSize=136.42 KB heapSize=390.06 KB 2024-11-25T05:54:55,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:54:55,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:54:55,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=A 2024-11-25T05:54:55,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-25T05:54:55,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=B 2024-11-25T05:54:55,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-25T05:54:55,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=C 2024-11-25T05:54:55,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-25T05:54:55,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/f811aebd015942ebb437e971dd0ba385 is 50, key is test_row_0/A:col10/1732514095435/Put/seqid=0 2024-11-25T05:54:55,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741840_1016 (size=14341) 2024-11-25T05:54:55,937 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/f811aebd015942ebb437e971dd0ba385 2024-11-25T05:54:55,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/123037e539c8470887975e967cb1b0c3 is 50, key is test_row_0/B:col10/1732514095435/Put/seqid=0 2024-11-25T05:54:55,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741841_1017 (size=12001) 2024-11-25T05:54:55,973 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/123037e539c8470887975e967cb1b0c3 2024-11-25T05:54:55,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:55,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514155913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:55,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514155920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:55,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:55,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:56,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:56,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/339ad02410d5431a816bf3e60061afbc is 50, key is test_row_0/C:col10/1732514095435/Put/seqid=0 2024-11-25T05:54:56,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514155994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:56,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:56,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514155997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:56,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:56,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514156001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:56,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:56,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741842_1018 (size=12001) 2024-11-25T05:54:56,059 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/339ad02410d5431a816bf3e60061afbc 2024-11-25T05:54:56,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/f811aebd015942ebb437e971dd0ba385 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/f811aebd015942ebb437e971dd0ba385 2024-11-25T05:54:56,085 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/f811aebd015942ebb437e971dd0ba385, entries=200, sequenceid=49, filesize=14.0 K 2024-11-25T05:54:56,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/123037e539c8470887975e967cb1b0c3 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/123037e539c8470887975e967cb1b0c3 2024-11-25T05:54:56,103 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/123037e539c8470887975e967cb1b0c3, entries=150, sequenceid=49, filesize=11.7 K 2024-11-25T05:54:56,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/339ad02410d5431a816bf3e60061afbc as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/339ad02410d5431a816bf3e60061afbc 2024-11-25T05:54:56,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:56,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514156103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:56,122 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/339ad02410d5431a816bf3e60061afbc, entries=150, sequenceid=49, filesize=11.7 K 2024-11-25T05:54:56,124 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~143.13 KB/146560, heapSize ~410.64 KB/420496, currentSize=53.67 KB/54960 for cf8ce3f4f3ad5ac4b5297b0ca903cd34 in 284ms, sequenceid=49, compaction requested=false 2024-11-25T05:54:56,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:56,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:54:56,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:56,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-25T05:54:56,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-25T05:54:56,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:54:56,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-25T05:54:56,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=A 2024-11-25T05:54:56,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:54:56,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=B 2024-11-25T05:54:56,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:54:56,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=C 2024-11-25T05:54:56,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:54:56,139 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-25T05:54:56,140 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 994 msec 2024-11-25T05:54:56,145 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.0280 sec 2024-11-25T05:54:56,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/bd9771db82994a15b536a0ccf907d1be is 50, key is test_row_0/A:col10/1732514095957/Put/seqid=0 2024-11-25T05:54:56,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741843_1019 (size=14341) 2024-11-25T05:54:56,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/bd9771db82994a15b536a0ccf907d1be 2024-11-25T05:54:56,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/5eabb2c3f6db41018dfea8bec92d1f86 is 50, key is test_row_0/B:col10/1732514095957/Put/seqid=0 2024-11-25T05:54:56,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741844_1020 (size=12001) 2024-11-25T05:54:56,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/5eabb2c3f6db41018dfea8bec92d1f86 2024-11-25T05:54:56,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T05:54:56,273 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:54:56,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/2869d05cfca94ac3ad84535cc9ae9c8a is 50, key is test_row_0/C:col10/1732514095957/Put/seqid=0 2024-11-25T05:54:56,279 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:54:56,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-25T05:54:56,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-25T05:54:56,284 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:54:56,286 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:54:56,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732514096218, max timestamp=1732514096268Num uniques 859; 2024-11-25T05:54:56,286 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:54:56,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T05:54:56,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732514096218, max timestamp=1732514096268Num uniques 859; 2024-11-25T05:54:56,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T05:54:56,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732514096218, max timestamp=1732514096268Num uniques 859; 2024-11-25T05:54:56,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-25T05:54:56,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741845_1021 (size=12001) 2024-11-25T05:54:56,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:56,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514156336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:56,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:56,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:56,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514156337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:56,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:56,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:56,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514156343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:56,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:56,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:56,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514156346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:56,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:56,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:56,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514156356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:56,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:56,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-25T05:54:56,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-25T05:54:56,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:56,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:54:56,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:56,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:56,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:56,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:56,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:56,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514156464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:56,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:56,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:56,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514156464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:56,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:56,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514156464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:56,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:56,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:56,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:56,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514156473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:56,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:56,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:56,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514156487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:56,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:56,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-25T05:54:56,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:56,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:54:56,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:56,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:56,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:56,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:56,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-25T05:54:56,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:56,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514156684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:56,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:56,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:56,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514156685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:56,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514156685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:56,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:56,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:56,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:56,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:56,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514156695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:56,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:56,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:56,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514156703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:56,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:56,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:56,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/2869d05cfca94ac3ad84535cc9ae9c8a 2024-11-25T05:54:56,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/bd9771db82994a15b536a0ccf907d1be as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/bd9771db82994a15b536a0ccf907d1be 2024-11-25T05:54:56,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/bd9771db82994a15b536a0ccf907d1be, entries=200, sequenceid=61, filesize=14.0 K 2024-11-25T05:54:56,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/5eabb2c3f6db41018dfea8bec92d1f86 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/5eabb2c3f6db41018dfea8bec92d1f86 2024-11-25T05:54:56,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-25T05:54:56,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:56,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:54:56,756 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/5eabb2c3f6db41018dfea8bec92d1f86, entries=150, sequenceid=61, filesize=11.7 K 2024-11-25T05:54:56,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:56,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:56,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:56,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:56,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/2869d05cfca94ac3ad84535cc9ae9c8a as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/2869d05cfca94ac3ad84535cc9ae9c8a 2024-11-25T05:54:56,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/2869d05cfca94ac3ad84535cc9ae9c8a, entries=150, sequenceid=61, filesize=11.7 K 2024-11-25T05:54:56,780 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=118.13 KB/120965 for cf8ce3f4f3ad5ac4b5297b0ca903cd34 in 644ms, sequenceid=61, compaction requested=true 2024-11-25T05:54:56,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:54:56,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:54:56,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:54:56,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:54:56,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:54:56,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:54:56,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:54:56,785 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:54:56,786 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:54:56,790 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:54:56,791 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:54:56,793 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/B is initiating minor compaction (all files) 2024-11-25T05:54:56,793 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/B in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:56,794 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/6d6bdc853a7d487ea551c72ca4d240ae, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/123037e539c8470887975e967cb1b0c3, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/5eabb2c3f6db41018dfea8bec92d1f86] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=35.2 K 2024-11-25T05:54:56,797 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d6bdc853a7d487ea551c72ca4d240ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732514095123 2024-11-25T05:54:56,799 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/A is initiating minor compaction (all files) 2024-11-25T05:54:56,799 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/A in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:56,799 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 123037e539c8470887975e967cb1b0c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732514095435 2024-11-25T05:54:56,799 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/3246d71eaa544a84af4849a86ce4a65c, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/f811aebd015942ebb437e971dd0ba385, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/bd9771db82994a15b536a0ccf907d1be] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=44.3 K 2024-11-25T05:54:56,801 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3246d71eaa544a84af4849a86ce4a65c, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732514095123 2024-11-25T05:54:56,802 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 5eabb2c3f6db41018dfea8bec92d1f86, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732514095856 2024-11-25T05:54:56,806 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting f811aebd015942ebb437e971dd0ba385, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732514095435 2024-11-25T05:54:56,807 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd9771db82994a15b536a0ccf907d1be, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732514095856 2024-11-25T05:54:56,846 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#A#compaction#10 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:54:56,846 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#B#compaction#9 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:54:56,847 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/ee565dbed7e74bb89fc5604e76bfc8c4 is 50, key is test_row_0/A:col10/1732514095957/Put/seqid=0 2024-11-25T05:54:56,847 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/b266dedc818e429abe71873f430ee80b is 50, key is test_row_0/B:col10/1732514095957/Put/seqid=0 2024-11-25T05:54:56,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741846_1022 (size=12104) 2024-11-25T05:54:56,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741847_1023 (size=12104) 2024-11-25T05:54:56,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-25T05:54:56,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:56,912 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34 3/3 column families, dataSize=118.13 KB heapSize=334.46 KB 2024-11-25T05:54:56,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=A 2024-11-25T05:54:56,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:54:56,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=B 2024-11-25T05:54:56,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:54:56,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=C 2024-11-25T05:54:56,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-25T05:54:56,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:54:56,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/4cfed5fe79764b748be44d695976755f is 50, key is test_row_0/A:col10/1732514096332/Put/seqid=0 2024-11-25T05:54:56,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741848_1024 (size=12001) 2024-11-25T05:54:56,941 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/4cfed5fe79764b748be44d695976755f 2024-11-25T05:54:56,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/e66baecc70384ea08d537ff7fa87f5da is 50, key is test_row_0/B:col10/1732514096332/Put/seqid=0 2024-11-25T05:54:57,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741849_1025 (size=12001) 2024-11-25T05:54:57,005 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/e66baecc70384ea08d537ff7fa87f5da 2024-11-25T05:54:57,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:54:57,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:54:57,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/a3055e98768f4aefbba68629f83a2585 is 50, key is test_row_0/C:col10/1732514096332/Put/seqid=0 2024-11-25T05:54:57,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:57,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514157026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:57,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:57,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514157039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:57,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:57,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514157044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:57,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:57,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:57,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:57,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514157045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:57,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:57,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741850_1026 (size=12001) 2024-11-25T05:54:57,063 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=37.62 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/a3055e98768f4aefbba68629f83a2585 2024-11-25T05:54:57,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514157051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:57,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:57,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/4cfed5fe79764b748be44d695976755f as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/4cfed5fe79764b748be44d695976755f 2024-11-25T05:54:57,104 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/4cfed5fe79764b748be44d695976755f, entries=150, sequenceid=95, filesize=11.7 K 2024-11-25T05:54:57,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/e66baecc70384ea08d537ff7fa87f5da as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/e66baecc70384ea08d537ff7fa87f5da 2024-11-25T05:54:57,133 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/e66baecc70384ea08d537ff7fa87f5da, entries=150, sequenceid=95, filesize=11.7 K 2024-11-25T05:54:57,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/a3055e98768f4aefbba68629f83a2585 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/a3055e98768f4aefbba68629f83a2585 2024-11-25T05:54:57,154 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/a3055e98768f4aefbba68629f83a2585, entries=150, sequenceid=95, filesize=11.7 K 2024-11-25T05:54:57,157 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~118.13 KB/120965, heapSize ~334.41 KB/342440, currentSize=67.09 KB/68700 for cf8ce3f4f3ad5ac4b5297b0ca903cd34 in 246ms, sequenceid=95, compaction requested=true 2024-11-25T05:54:57,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:54:57,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:57,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-25T05:54:57,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-25T05:54:57,166 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-25T05:54:57,166 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-25T05:54:57,166 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 875 msec 2024-11-25T05:54:57,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=A 2024-11-25T05:54:57,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:54:57,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=B 2024-11-25T05:54:57,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:54:57,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=C 2024-11-25T05:54:57,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:54:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:54:57,172 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 889 msec 2024-11-25T05:54:57,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/2779c5bef419405bafc6deb24302fade is 50, key is test_row_0/A:col10/1732514097034/Put/seqid=0 2024-11-25T05:54:57,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514157207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:57,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:57,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514157208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:57,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:57,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514157212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:57,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:57,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514157214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:57,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:57,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514157217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:57,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741851_1027 (size=14341) 2024-11-25T05:54:57,283 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/b266dedc818e429abe71873f430ee80b as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/b266dedc818e429abe71873f430ee80b 2024-11-25T05:54:57,285 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/ee565dbed7e74bb89fc5604e76bfc8c4 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/ee565dbed7e74bb89fc5604e76bfc8c4 2024-11-25T05:54:57,318 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/A of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into ee565dbed7e74bb89fc5604e76bfc8c4(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:54:57,318 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:54:57,320 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/B of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into b266dedc818e429abe71873f430ee80b(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:54:57,320 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:54:57,321 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/A, priority=13, startTime=1732514096781; duration=0sec 2024-11-25T05:54:57,321 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/B, priority=13, startTime=1732514096784; duration=0sec 2024-11-25T05:54:57,321 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:54:57,322 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:A 2024-11-25T05:54:57,322 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T05:54:57,326 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:54:57,326 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:B 2024-11-25T05:54:57,327 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T05:54:57,327 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/C is initiating minor compaction (all files) 2024-11-25T05:54:57,328 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/C in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:57,328 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/7abcf038322249a0b81f5e3fca326f2a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/339ad02410d5431a816bf3e60061afbc, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/2869d05cfca94ac3ad84535cc9ae9c8a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/a3055e98768f4aefbba68629f83a2585] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=46.9 K 2024-11-25T05:54:57,329 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7abcf038322249a0b81f5e3fca326f2a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732514095123 2024-11-25T05:54:57,332 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 339ad02410d5431a816bf3e60061afbc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732514095435 2024-11-25T05:54:57,335 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2869d05cfca94ac3ad84535cc9ae9c8a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732514095856 2024-11-25T05:54:57,336 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3055e98768f4aefbba68629f83a2585, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732514096319 2024-11-25T05:54:57,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514157341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:57,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:57,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514157342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:57,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:57,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514157341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:57,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:57,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514157343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:57,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:57,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514157343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:57,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:57,383 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#C#compaction#15 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:54:57,384 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/8520dae074d2462fa9fc8da3391e67f8 is 50, key is test_row_0/C:col10/1732514096332/Put/seqid=0 2024-11-25T05:54:57,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741852_1028 (size=12139) 2024-11-25T05:54:57,415 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/8520dae074d2462fa9fc8da3391e67f8 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/8520dae074d2462fa9fc8da3391e67f8 2024-11-25T05:54:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-25T05:54:57,425 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:54:57,426 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:54:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-25T05:54:57,432 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:54:57,435 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:54:57,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:54:57,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T05:54:57,439 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/C of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into 8520dae074d2462fa9fc8da3391e67f8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:54:57,440 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:54:57,440 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/C, priority=12, startTime=1732514096784; duration=0sec 2024-11-25T05:54:57,440 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:54:57,440 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:C 2024-11-25T05:54:57,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T05:54:57,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:57,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514157552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:57,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:57,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:57,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514157552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:57,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:57,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:57,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514157562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:57,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:57,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:57,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514157562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:57,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:57,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:57,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514157562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:57,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:57,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-25T05:54:57,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:57,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:54:57,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:57,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:57,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:57,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:57,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/2779c5bef419405bafc6deb24302fade 2024-11-25T05:54:57,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/6d06f52b20dc4599a2f9438526abce10 is 50, key is test_row_0/B:col10/1732514097034/Put/seqid=0 2024-11-25T05:54:57,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741853_1029 (size=12001) 2024-11-25T05:54:57,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/6d06f52b20dc4599a2f9438526abce10 2024-11-25T05:54:57,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/b262418a9add470db153964485b3ffd7 is 50, key is test_row_0/C:col10/1732514097034/Put/seqid=0 2024-11-25T05:54:57,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741854_1030 (size=12001) 2024-11-25T05:54:57,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/b262418a9add470db153964485b3ffd7 2024-11-25T05:54:57,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/2779c5bef419405bafc6deb24302fade as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/2779c5bef419405bafc6deb24302fade 2024-11-25T05:54:57,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-25T05:54:57,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:57,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:54:57,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:57,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:57,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:57,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:57,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T05:54:57,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/2779c5bef419405bafc6deb24302fade, entries=200, sequenceid=109, filesize=14.0 K 2024-11-25T05:54:57,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/6d06f52b20dc4599a2f9438526abce10 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/6d06f52b20dc4599a2f9438526abce10 2024-11-25T05:54:57,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/6d06f52b20dc4599a2f9438526abce10, entries=150, sequenceid=109, filesize=11.7 K 2024-11-25T05:54:57,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/b262418a9add470db153964485b3ffd7 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/b262418a9add470db153964485b3ffd7 2024-11-25T05:54:57,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/b262418a9add470db153964485b3ffd7, entries=150, sequenceid=109, filesize=11.7 K 2024-11-25T05:54:57,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for cf8ce3f4f3ad5ac4b5297b0ca903cd34 in 644ms, sequenceid=109, compaction requested=true 2024-11-25T05:54:57,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:54:57,810 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:54:57,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:54:57,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:54:57,810 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:54:57,812 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:54:57,812 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/A is initiating minor compaction (all files) 2024-11-25T05:54:57,813 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:54:57,813 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/A in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:57,813 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/B is initiating minor compaction (all files) 2024-11-25T05:54:57,813 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/B in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:57,813 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/ee565dbed7e74bb89fc5604e76bfc8c4, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/4cfed5fe79764b748be44d695976755f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/2779c5bef419405bafc6deb24302fade] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=37.5 K 2024-11-25T05:54:57,813 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/b266dedc818e429abe71873f430ee80b, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/e66baecc70384ea08d537ff7fa87f5da, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/6d06f52b20dc4599a2f9438526abce10] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=35.3 K 2024-11-25T05:54:57,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:54:57,814 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting b266dedc818e429abe71873f430ee80b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732514095856 2024-11-25T05:54:57,814 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee565dbed7e74bb89fc5604e76bfc8c4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732514095856 2024-11-25T05:54:57,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:54:57,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:54:57,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:54:57,815 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cfed5fe79764b748be44d695976755f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732514096319 2024-11-25T05:54:57,815 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting e66baecc70384ea08d537ff7fa87f5da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732514096319 2024-11-25T05:54:57,816 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2779c5bef419405bafc6deb24302fade, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732514097034 2024-11-25T05:54:57,816 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d06f52b20dc4599a2f9438526abce10, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732514097034 2024-11-25T05:54:57,838 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#B#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:54:57,839 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/77d09ef194e041c7b981ae3b8975ff17 is 50, key is test_row_0/B:col10/1732514097034/Put/seqid=0 2024-11-25T05:54:57,843 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#A#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:54:57,844 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/f0c6fc1812d34dd8ae8c1322d28fe729 is 50, key is test_row_0/A:col10/1732514097034/Put/seqid=0 2024-11-25T05:54:57,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:54:57,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-25T05:54:57,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=A 2024-11-25T05:54:57,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:54:57,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=B 2024-11-25T05:54:57,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:54:57,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=C 2024-11-25T05:54:57,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:54:57,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741855_1031 (size=12207) 2024-11-25T05:54:57,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-25T05:54:57,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:57,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:54:57,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:57,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:57,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:57,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/65f8e7ee43134361919bedb6f71ec2cd is 50, key is test_row_0/A:col10/1732514097863/Put/seqid=0 2024-11-25T05:54:57,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:57,909 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/77d09ef194e041c7b981ae3b8975ff17 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/77d09ef194e041c7b981ae3b8975ff17 2024-11-25T05:54:57,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741856_1032 (size=12207) 2024-11-25T05:54:57,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:57,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514157904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:57,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:57,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:57,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514157909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:57,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:57,923 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/B of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into 77d09ef194e041c7b981ae3b8975ff17(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:54:57,924 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:54:57,924 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/B, priority=13, startTime=1732514097810; duration=0sec 2024-11-25T05:54:57,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:57,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514157910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:57,924 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:54:57,924 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:B 2024-11-25T05:54:57,924 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-25T05:54:57,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:57,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:57,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514157917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:57,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:57,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:57,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514157920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:57,927 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-25T05:54:57,927 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-25T05:54:57,927 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. because compaction request was cancelled 2024-11-25T05:54:57,927 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:C 2024-11-25T05:54:57,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:57,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:57,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741857_1033 (size=14391) 2024-11-25T05:54:57,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/65f8e7ee43134361919bedb6f71ec2cd 2024-11-25T05:54:57,949 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T05:54:57,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/af6102ff989745dca5794d9746b1176c is 50, key is test_row_0/B:col10/1732514097863/Put/seqid=0 2024-11-25T05:54:57,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741858_1034 (size=12051) 2024-11-25T05:54:58,014 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-25T05:54:58,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:58,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514158032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:58,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:58,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514158033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:58,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:58,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:58,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:58,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514158033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:58,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:58,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:58,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514158033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:58,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:58,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514158033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:58,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:58,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:58,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:58,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:58,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:58,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:58,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-25T05:54:58,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T05:54:58,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:58,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:54:58,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:58,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:58,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:58,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:58,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-25T05:54:58,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:58,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:54:58,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:58,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:58,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:58,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:58,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:58,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514158243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:58,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:58,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:58,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:58,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:58,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514158243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:58,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:58,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:58,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:58,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:58,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514158243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:58,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:58,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:58,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:58,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:58,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514158253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:58,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:58,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:58,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:58,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:58,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514158254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:58,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:58,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:58,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:58,336 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/f0c6fc1812d34dd8ae8c1322d28fe729 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/f0c6fc1812d34dd8ae8c1322d28fe729 2024-11-25T05:54:58,349 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/A of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into f0c6fc1812d34dd8ae8c1322d28fe729(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:54:58,349 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:54:58,349 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/A, priority=13, startTime=1732514097810; duration=0sec 2024-11-25T05:54:58,350 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:54:58,350 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:A 2024-11-25T05:54:58,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-25T05:54:58,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:58,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:54:58,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:58,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:58,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:58,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:58,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/af6102ff989745dca5794d9746b1176c 2024-11-25T05:54:58,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/0d217429f2da4e598df28a86cfd4847e is 50, key is test_row_0/C:col10/1732514097863/Put/seqid=0 2024-11-25T05:54:58,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741859_1035 (size=12051) 2024-11-25T05:54:58,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-25T05:54:58,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:58,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:54:58,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:58,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:58,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:58,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:58,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:58,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514158552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:58,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:58,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514158553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:58,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:58,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:58,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:58,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:58,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:58,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:58,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:58,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514158552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:58,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:58,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514158562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:58,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:58,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:58,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:58,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:58,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514158563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:58,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:58,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:58,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:58,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:58,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:58,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:58,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T05:54:58,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-25T05:54:58,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:58,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:54:58,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:58,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:58,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:58,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:58,765 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-25T05:54:58,766 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-25T05:54:58,768 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T05:54:58,768 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-25T05:54:58,769 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-25T05:54:58,769 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-25T05:54:58,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-25T05:54:58,771 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-25T05:54:58,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-25T05:54:58,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:58,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:54:58,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:58,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:58,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:58,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:54:58,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/0d217429f2da4e598df28a86cfd4847e 2024-11-25T05:54:58,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/65f8e7ee43134361919bedb6f71ec2cd as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/65f8e7ee43134361919bedb6f71ec2cd 2024-11-25T05:54:58,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/65f8e7ee43134361919bedb6f71ec2cd, entries=200, sequenceid=131, filesize=14.1 K 2024-11-25T05:54:58,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/af6102ff989745dca5794d9746b1176c as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/af6102ff989745dca5794d9746b1176c 2024-11-25T05:54:58,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/af6102ff989745dca5794d9746b1176c, entries=150, sequenceid=131, filesize=11.8 K 2024-11-25T05:54:58,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/0d217429f2da4e598df28a86cfd4847e as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/0d217429f2da4e598df28a86cfd4847e 2024-11-25T05:54:58,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/0d217429f2da4e598df28a86cfd4847e, entries=150, sequenceid=131, filesize=11.8 K 2024-11-25T05:54:58,962 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for cf8ce3f4f3ad5ac4b5297b0ca903cd34 in 1093ms, sequenceid=131, compaction requested=true 2024-11-25T05:54:58,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:54:58,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:54:58,962 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-25T05:54:58,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:54:58,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:54:58,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:54:58,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:54:58,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T05:54:58,963 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:54:58,965 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-25T05:54:58,965 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-25T05:54:58,965 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. because compaction request was cancelled 2024-11-25T05:54:58,965 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:A 2024-11-25T05:54:58,965 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-25T05:54:58,966 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36191 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:54:58,966 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/C is initiating minor compaction (all files) 2024-11-25T05:54:58,966 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/C in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:58,966 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/8520dae074d2462fa9fc8da3391e67f8, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/b262418a9add470db153964485b3ffd7, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/0d217429f2da4e598df28a86cfd4847e] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=35.3 K 2024-11-25T05:54:58,967 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8520dae074d2462fa9fc8da3391e67f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732514096319 2024-11-25T05:54:58,968 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-25T05:54:58,968 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting b262418a9add470db153964485b3ffd7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732514097034 2024-11-25T05:54:58,968 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-25T05:54:58,968 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. because compaction request was cancelled 2024-11-25T05:54:58,968 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:B 2024-11-25T05:54:58,969 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d217429f2da4e598df28a86cfd4847e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732514097204 2024-11-25T05:54:59,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-25T05:54:59,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:59,006 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-25T05:54:59,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=A 2024-11-25T05:54:59,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:54:59,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=B 2024-11-25T05:54:59,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:54:59,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=C 2024-11-25T05:54:59,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:54:59,013 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#C#compaction#23 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:54:59,014 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/0dd70a2a3a164cb9bdff950b529a696a is 50, key is test_row_0/C:col10/1732514097863/Put/seqid=0 2024-11-25T05:54:59,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/c39e8bb7b4bb415eb04df4e3b5e035e0 is 50, key is test_row_0/A:col10/1732514097917/Put/seqid=0 2024-11-25T05:54:59,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741860_1036 (size=12291) 2024-11-25T05:54:59,069 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/0dd70a2a3a164cb9bdff950b529a696a as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/0dd70a2a3a164cb9bdff950b529a696a 2024-11-25T05:54:59,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:54:59,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:54:59,088 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/C of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into 0dd70a2a3a164cb9bdff950b529a696a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:54:59,088 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:54:59,088 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/C, priority=13, startTime=1732514098962; duration=0sec 2024-11-25T05:54:59,089 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:54:59,089 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:C 2024-11-25T05:54:59,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741861_1037 (size=12151) 2024-11-25T05:54:59,100 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/c39e8bb7b4bb415eb04df4e3b5e035e0 2024-11-25T05:54:59,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/5100763999c940368b315970163e446a is 50, key is test_row_0/B:col10/1732514097917/Put/seqid=0 2024-11-25T05:54:59,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:59,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514159124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:59,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:59,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:59,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514159131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:59,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:59,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514159133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:59,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:59,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:59,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:59,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514159137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:59,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:59,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:59,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514159139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:59,152 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,152 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,152 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:59,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741862_1038 (size=12151) 2024-11-25T05:54:59,171 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/5100763999c940368b315970163e446a 2024-11-25T05:54:59,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/4374630a9b1348feada0fb5a03f02317 is 50, key is test_row_0/C:col10/1732514097917/Put/seqid=0 2024-11-25T05:54:59,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741863_1039 (size=12151) 2024-11-25T05:54:59,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:59,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514159243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:59,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:59,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:59,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514159252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:59,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:59,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514159252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:59,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:59,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514159252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:59,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:59,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:59,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:59,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514159262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:59,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:59,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:59,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514159452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:59,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:59,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:59,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514159463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:59,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:59,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:59,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514159474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:59,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:59,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:59,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514159474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:59,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:59,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:59,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514159483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:59,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:59,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T05:54:59,643 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/4374630a9b1348feada0fb5a03f02317 2024-11-25T05:54:59,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/c39e8bb7b4bb415eb04df4e3b5e035e0 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/c39e8bb7b4bb415eb04df4e3b5e035e0 2024-11-25T05:54:59,677 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/c39e8bb7b4bb415eb04df4e3b5e035e0, entries=150, sequenceid=146, filesize=11.9 K 2024-11-25T05:54:59,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/5100763999c940368b315970163e446a as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/5100763999c940368b315970163e446a 2024-11-25T05:54:59,693 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/5100763999c940368b315970163e446a, entries=150, sequenceid=146, filesize=11.9 K 2024-11-25T05:54:59,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/4374630a9b1348feada0fb5a03f02317 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/4374630a9b1348feada0fb5a03f02317 2024-11-25T05:54:59,725 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/4374630a9b1348feada0fb5a03f02317, entries=150, sequenceid=146, filesize=11.9 K 2024-11-25T05:54:59,729 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for cf8ce3f4f3ad5ac4b5297b0ca903cd34 in 723ms, sequenceid=146, compaction requested=true 2024-11-25T05:54:59,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:54:59,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:54:59,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-25T05:54:59,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-25T05:54:59,742 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-25T05:54:59,742 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2970 sec 2024-11-25T05:54:59,749 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.3180 sec 2024-11-25T05:54:59,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:54:59,783 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-25T05:54:59,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=A 2024-11-25T05:54:59,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:54:59,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=B 2024-11-25T05:54:59,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:54:59,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=C 2024-11-25T05:54:59,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:54:59,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/0d72988bf0404061b94b7be810b8a473 is 50, key is test_row_0/A:col10/1732514099135/Put/seqid=0 2024-11-25T05:54:59,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:59,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514159825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:59,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:59,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514159833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:59,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:59,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:59,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:59,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514159836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:59,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:59,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514159828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:59,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:59,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741864_1040 (size=12151) 2024-11-25T05:54:59,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:59,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514159838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:59,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:59,873 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/0d72988bf0404061b94b7be810b8a473 2024-11-25T05:54:59,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/cfa41c4410cf48dc97a4a2cbb507fb85 is 50, key is test_row_0/B:col10/1732514099135/Put/seqid=0 2024-11-25T05:54:59,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:59,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514159942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:59,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:59,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741865_1041 (size=12151) 2024-11-25T05:54:59,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/cfa41c4410cf48dc97a4a2cbb507fb85 2024-11-25T05:54:59,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:59,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514159952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:59,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:59,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:59,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514159953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:59,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:59,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:59,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514159953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:54:59,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:54:59,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:54:59,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:54:59,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514159972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:00,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/65fd1217fd81459b9e5ed62d153bd0a7 is 50, key is test_row_0/C:col10/1732514099135/Put/seqid=0 2024-11-25T05:55:00,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:00,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741866_1042 (size=12151) 2024-11-25T05:55:00,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/65fd1217fd81459b9e5ed62d153bd0a7 2024-11-25T05:55:00,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/0d72988bf0404061b94b7be810b8a473 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/0d72988bf0404061b94b7be810b8a473 2024-11-25T05:55:00,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/0d72988bf0404061b94b7be810b8a473, entries=150, sequenceid=167, filesize=11.9 K 2024-11-25T05:55:00,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/cfa41c4410cf48dc97a4a2cbb507fb85 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/cfa41c4410cf48dc97a4a2cbb507fb85 2024-11-25T05:55:00,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/cfa41c4410cf48dc97a4a2cbb507fb85, entries=150, sequenceid=167, filesize=11.9 K 2024-11-25T05:55:00,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/65fd1217fd81459b9e5ed62d153bd0a7 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/65fd1217fd81459b9e5ed62d153bd0a7 2024-11-25T05:55:00,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/65fd1217fd81459b9e5ed62d153bd0a7, entries=150, sequenceid=167, filesize=11.9 K 2024-11-25T05:55:00,114 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for cf8ce3f4f3ad5ac4b5297b0ca903cd34 in 331ms, sequenceid=167, compaction requested=true 2024-11-25T05:55:00,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:00,114 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T05:55:00,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:55:00,116 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50900 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T05:55:00,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:00,117 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/A is initiating minor compaction (all files) 2024-11-25T05:55:00,117 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T05:55:00,117 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/A in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:00,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:55:00,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:00,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:55:00,117 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/f0c6fc1812d34dd8ae8c1322d28fe729, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/65f8e7ee43134361919bedb6f71ec2cd, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/c39e8bb7b4bb415eb04df4e3b5e035e0, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/0d72988bf0404061b94b7be810b8a473] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=49.7 K 2024-11-25T05:55:00,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:00,118 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0c6fc1812d34dd8ae8c1322d28fe729, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732514097034 2024-11-25T05:55:00,119 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65f8e7ee43134361919bedb6f71ec2cd, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732514097204 2024-11-25T05:55:00,119 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting c39e8bb7b4bb415eb04df4e3b5e035e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732514097900 2024-11-25T05:55:00,120 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d72988bf0404061b94b7be810b8a473, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732514099133 2024-11-25T05:55:00,126 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48560 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T05:55:00,129 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/B is initiating minor compaction (all files) 2024-11-25T05:55:00,129 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/B in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:00,129 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/77d09ef194e041c7b981ae3b8975ff17, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/af6102ff989745dca5794d9746b1176c, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/5100763999c940368b315970163e446a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/cfa41c4410cf48dc97a4a2cbb507fb85] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=47.4 K 2024-11-25T05:55:00,130 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 77d09ef194e041c7b981ae3b8975ff17, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732514097034 2024-11-25T05:55:00,131 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting af6102ff989745dca5794d9746b1176c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732514097204 2024-11-25T05:55:00,132 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 5100763999c940368b315970163e446a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732514097900 2024-11-25T05:55:00,133 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting cfa41c4410cf48dc97a4a2cbb507fb85, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732514099133 2024-11-25T05:55:00,153 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#A#compaction#30 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:00,154 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/d755bf3560254e28833cbe8f8f216aba is 50, key is test_row_0/A:col10/1732514099135/Put/seqid=0 2024-11-25T05:55:00,154 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#B#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:00,155 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/71709038fb994bd5acbd9a5669455102 is 50, key is test_row_0/B:col10/1732514099135/Put/seqid=0 2024-11-25T05:55:00,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:55:00,163 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-25T05:55:00,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=A 2024-11-25T05:55:00,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:00,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=B 2024-11-25T05:55:00,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:00,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=C 2024-11-25T05:55:00,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:00,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/7dbbcadfea1b4233ab46cf36e22e911e is 50, key is test_row_0/A:col10/1732514099833/Put/seqid=0 2024-11-25T05:55:00,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741867_1043 (size=12493) 2024-11-25T05:55:00,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741868_1044 (size=12493) 2024-11-25T05:55:00,231 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/71709038fb994bd5acbd9a5669455102 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/71709038fb994bd5acbd9a5669455102 2024-11-25T05:55:00,250 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/B of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into 71709038fb994bd5acbd9a5669455102(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:00,251 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:00,251 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/B, priority=12, startTime=1732514100117; duration=0sec 2024-11-25T05:55:00,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741869_1045 (size=14541) 2024-11-25T05:55:00,253 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:00,253 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:B 2024-11-25T05:55:00,253 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/7dbbcadfea1b4233ab46cf36e22e911e 2024-11-25T05:55:00,253 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:00,260 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36593 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:00,260 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/C is initiating minor compaction (all files) 2024-11-25T05:55:00,260 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/C in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:00,260 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/0dd70a2a3a164cb9bdff950b529a696a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/4374630a9b1348feada0fb5a03f02317, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/65fd1217fd81459b9e5ed62d153bd0a7] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=35.7 K 2024-11-25T05:55:00,261 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dd70a2a3a164cb9bdff950b529a696a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732514097204 2024-11-25T05:55:00,261 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 4374630a9b1348feada0fb5a03f02317, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732514097900 2024-11-25T05:55:00,262 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 65fd1217fd81459b9e5ed62d153bd0a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732514099133 2024-11-25T05:55:00,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:00,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514160250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:00,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:00,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:00,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514160253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:00,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:00,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:00,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514160263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:00,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:00,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/3cd5b80953fe4c54a7cf822f15d4dfdc is 50, key is test_row_0/B:col10/1732514099833/Put/seqid=0 2024-11-25T05:55:00,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:00,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514160266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:00,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:00,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514160264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:00,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:00,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:00,313 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#C#compaction#34 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:00,321 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/981c73fee7a84a1f907c69065d5d110e is 50, key is test_row_0/C:col10/1732514099135/Put/seqid=0 2024-11-25T05:55:00,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741870_1046 (size=12151) 2024-11-25T05:55:00,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/3cd5b80953fe4c54a7cf822f15d4dfdc 2024-11-25T05:55:00,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741871_1047 (size=12493) 2024-11-25T05:55:00,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/a522bd8b97bf4f598f9fd6586007f598 is 50, key is test_row_0/C:col10/1732514099833/Put/seqid=0 2024-11-25T05:55:00,373 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/981c73fee7a84a1f907c69065d5d110e as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/981c73fee7a84a1f907c69065d5d110e 2024-11-25T05:55:00,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:00,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514160373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:00,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:00,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514160373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:00,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:00,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:00,387 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/C of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into 981c73fee7a84a1f907c69065d5d110e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:00,388 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:00,388 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/C, priority=13, startTime=1732514100117; duration=0sec 2024-11-25T05:55:00,388 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:00,388 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:C 2024-11-25T05:55:00,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:00,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514160393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:00,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:00,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:00,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514160393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:00,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:00,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:00,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514160393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:00,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:00,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741872_1048 (size=12151) 2024-11-25T05:55:00,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/a522bd8b97bf4f598f9fd6586007f598 2024-11-25T05:55:00,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/7dbbcadfea1b4233ab46cf36e22e911e as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/7dbbcadfea1b4233ab46cf36e22e911e 2024-11-25T05:55:00,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/7dbbcadfea1b4233ab46cf36e22e911e, entries=200, sequenceid=181, filesize=14.2 K 2024-11-25T05:55:00,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/3cd5b80953fe4c54a7cf822f15d4dfdc as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/3cd5b80953fe4c54a7cf822f15d4dfdc 2024-11-25T05:55:00,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/3cd5b80953fe4c54a7cf822f15d4dfdc, entries=150, sequenceid=181, filesize=11.9 K 2024-11-25T05:55:00,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/a522bd8b97bf4f598f9fd6586007f598 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/a522bd8b97bf4f598f9fd6586007f598 2024-11-25T05:55:00,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/a522bd8b97bf4f598f9fd6586007f598, entries=150, sequenceid=181, filesize=11.9 K 2024-11-25T05:55:00,458 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for cf8ce3f4f3ad5ac4b5297b0ca903cd34 in 296ms, sequenceid=181, compaction requested=false 2024-11-25T05:55:00,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:00,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:55:00,590 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-25T05:55:00,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=A 2024-11-25T05:55:00,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:00,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=B 2024-11-25T05:55:00,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:00,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=C 2024-11-25T05:55:00,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:00,615 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/d755bf3560254e28833cbe8f8f216aba as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/d755bf3560254e28833cbe8f8f216aba 2024-11-25T05:55:00,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/7cce169531cc478bab68412d7ca44456 is 50, key is test_row_0/A:col10/1732514100586/Put/seqid=0 2024-11-25T05:55:00,634 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/A of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into d755bf3560254e28833cbe8f8f216aba(size=12.2 K), total size for store is 26.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:00,634 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:00,634 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/A, priority=12, startTime=1732514100114; duration=0sec 2024-11-25T05:55:00,635 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:00,635 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:A 2024-11-25T05:55:00,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:00,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514160639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:00,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:00,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:00,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514160647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:00,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:00,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:00,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:00,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514160652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:00,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:00,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741873_1049 (size=16931) 2024-11-25T05:55:00,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:00,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/7cce169531cc478bab68412d7ca44456 2024-11-25T05:55:00,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514160655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:00,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:00,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514160641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:00,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:00,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/6cc728eafdb1499eb102e9cb98e20479 is 50, key is test_row_0/B:col10/1732514100586/Put/seqid=0 2024-11-25T05:55:00,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741874_1050 (size=12151) 2024-11-25T05:55:00,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/6cc728eafdb1499eb102e9cb98e20479 2024-11-25T05:55:00,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:00,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514160765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:00,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:00,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/696b6c48962c4954a73aa38f35bc3929 is 50, key is test_row_0/C:col10/1732514100586/Put/seqid=0 2024-11-25T05:55:00,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:00,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514160765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:00,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:00,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:00,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514160773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:00,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:00,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514160774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:00,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:00,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:00,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:00,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514160782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:00,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:00,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:00,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741875_1051 (size=12151) 2024-11-25T05:55:00,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/696b6c48962c4954a73aa38f35bc3929 2024-11-25T05:55:00,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/7cce169531cc478bab68412d7ca44456 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/7cce169531cc478bab68412d7ca44456 2024-11-25T05:55:00,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/7cce169531cc478bab68412d7ca44456, entries=250, sequenceid=203, filesize=16.5 K 2024-11-25T05:55:00,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/6cc728eafdb1499eb102e9cb98e20479 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/6cc728eafdb1499eb102e9cb98e20479 2024-11-25T05:55:00,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/6cc728eafdb1499eb102e9cb98e20479, entries=150, sequenceid=203, filesize=11.9 K 2024-11-25T05:55:00,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/696b6c48962c4954a73aa38f35bc3929 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/696b6c48962c4954a73aa38f35bc3929 2024-11-25T05:55:00,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/696b6c48962c4954a73aa38f35bc3929, entries=150, sequenceid=203, filesize=11.9 K 2024-11-25T05:55:00,883 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for cf8ce3f4f3ad5ac4b5297b0ca903cd34 in 291ms, sequenceid=203, compaction requested=true 2024-11-25T05:55:00,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:00,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:55:00,883 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:00,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:00,883 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:00,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:55:00,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:00,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:55:00,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:00,885 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:00,885 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/A is initiating minor compaction (all files) 2024-11-25T05:55:00,885 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/A in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:00,885 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/d755bf3560254e28833cbe8f8f216aba, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/7dbbcadfea1b4233ab46cf36e22e911e, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/7cce169531cc478bab68412d7ca44456] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=42.9 K 2024-11-25T05:55:00,885 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:00,886 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/B is initiating minor compaction (all files) 2024-11-25T05:55:00,886 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/B in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:00,886 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/71709038fb994bd5acbd9a5669455102, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/3cd5b80953fe4c54a7cf822f15d4dfdc, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/6cc728eafdb1499eb102e9cb98e20479] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=35.9 K 2024-11-25T05:55:00,886 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting d755bf3560254e28833cbe8f8f216aba, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732514099133 2024-11-25T05:55:00,887 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 71709038fb994bd5acbd9a5669455102, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732514099133 2024-11-25T05:55:00,887 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7dbbcadfea1b4233ab46cf36e22e911e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732514099829 2024-11-25T05:55:00,887 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cd5b80953fe4c54a7cf822f15d4dfdc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732514099831 2024-11-25T05:55:00,887 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cce169531cc478bab68412d7ca44456, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732514100250 2024-11-25T05:55:00,891 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cc728eafdb1499eb102e9cb98e20479, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732514100250 2024-11-25T05:55:00,908 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#A#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:00,908 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#B#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:00,909 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/4849b1b74f084ae1aa99fbd605ca2b02 is 50, key is test_row_0/A:col10/1732514100586/Put/seqid=0 2024-11-25T05:55:00,909 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/bb3ccdee8b344b43bd6826ad248f0702 is 50, key is test_row_0/B:col10/1732514100586/Put/seqid=0 2024-11-25T05:55:00,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741877_1053 (size=12595) 2024-11-25T05:55:00,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741876_1052 (size=12595) 2024-11-25T05:55:01,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-25T05:55:01,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=A 2024-11-25T05:55:01,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:01,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=B 2024-11-25T05:55:01,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:01,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=C 2024-11-25T05:55:01,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:01,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:55:01,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/a3c1a33264dc4388be036a7ecd7546a6 is 50, key is test_row_0/A:col10/1732514100650/Put/seqid=0 2024-11-25T05:55:01,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741878_1054 (size=12151) 2024-11-25T05:55:01,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/a3c1a33264dc4388be036a7ecd7546a6 2024-11-25T05:55:01,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/d851c78e5e4f40cd89bc3a3d5e0a4f3a is 50, key is test_row_0/B:col10/1732514100650/Put/seqid=0 2024-11-25T05:55:01,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:01,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514161039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:01,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:01,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514161044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:01,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:01,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:01,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514161046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:01,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:01,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:01,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:01,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:01,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514161041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:01,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514161054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:01,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:01,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:01,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741879_1055 (size=12151) 2024-11-25T05:55:01,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:01,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514161162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:01,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:01,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:01,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514161162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:01,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:01,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:01,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:01,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514161172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:01,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514161172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:01,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:01,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:01,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:01,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514161172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:01,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:01,348 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/4849b1b74f084ae1aa99fbd605ca2b02 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/4849b1b74f084ae1aa99fbd605ca2b02 2024-11-25T05:55:01,359 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/A of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into 4849b1b74f084ae1aa99fbd605ca2b02(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:01,359 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:01,359 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/A, priority=13, startTime=1732514100883; duration=0sec 2024-11-25T05:55:01,363 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:01,363 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:A 2024-11-25T05:55:01,365 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:01,371 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/bb3ccdee8b344b43bd6826ad248f0702 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/bb3ccdee8b344b43bd6826ad248f0702 2024-11-25T05:55:01,374 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:01,374 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/C is initiating minor compaction (all files) 2024-11-25T05:55:01,374 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/C in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:01,375 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/981c73fee7a84a1f907c69065d5d110e, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/a522bd8b97bf4f598f9fd6586007f598, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/696b6c48962c4954a73aa38f35bc3929] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=35.9 K 2024-11-25T05:55:01,376 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 981c73fee7a84a1f907c69065d5d110e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732514099133 2024-11-25T05:55:01,379 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting a522bd8b97bf4f598f9fd6586007f598, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732514099831 2024-11-25T05:55:01,380 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 696b6c48962c4954a73aa38f35bc3929, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732514100250 2024-11-25T05:55:01,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:01,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514161373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:01,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:01,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514161373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:01,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:01,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:01,388 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/B of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into bb3ccdee8b344b43bd6826ad248f0702(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:01,388 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:01,389 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/B, priority=13, startTime=1732514100883; duration=0sec 2024-11-25T05:55:01,389 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:01,389 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:B 2024-11-25T05:55:01,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:01,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514161393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:01,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:01,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514161393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:01,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:01,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514161393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:01,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:01,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:01,425 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#C#compaction#43 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:01,425 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/1447f666f6de4f2d84584b89ae5181ad is 50, key is test_row_0/C:col10/1732514100586/Put/seqid=0 2024-11-25T05:55:01,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741880_1056 (size=12595) 2024-11-25T05:55:01,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/d851c78e5e4f40cd89bc3a3d5e0a4f3a 2024-11-25T05:55:01,490 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/1447f666f6de4f2d84584b89ae5181ad as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/1447f666f6de4f2d84584b89ae5181ad 2024-11-25T05:55:01,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/39b1bcab2759400284c2fd359d0c4593 is 50, key is test_row_0/C:col10/1732514100650/Put/seqid=0 2024-11-25T05:55:01,512 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/C of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into 1447f666f6de4f2d84584b89ae5181ad(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:01,512 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:01,512 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/C, priority=13, startTime=1732514100884; duration=0sec 2024-11-25T05:55:01,513 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:01,513 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:C 2024-11-25T05:55:01,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741881_1057 (size=12151) 2024-11-25T05:55:01,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/39b1bcab2759400284c2fd359d0c4593 2024-11-25T05:55:01,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/a3c1a33264dc4388be036a7ecd7546a6 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/a3c1a33264dc4388be036a7ecd7546a6 2024-11-25T05:55:01,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/a3c1a33264dc4388be036a7ecd7546a6, entries=150, sequenceid=219, filesize=11.9 K 2024-11-25T05:55:01,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/d851c78e5e4f40cd89bc3a3d5e0a4f3a as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/d851c78e5e4f40cd89bc3a3d5e0a4f3a 2024-11-25T05:55:01,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/d851c78e5e4f40cd89bc3a3d5e0a4f3a, entries=150, sequenceid=219, filesize=11.9 K 2024-11-25T05:55:01,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/39b1bcab2759400284c2fd359d0c4593 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/39b1bcab2759400284c2fd359d0c4593 2024-11-25T05:55:01,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T05:55:01,593 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:55:01,594 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:55:01,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-25T05:55:01,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/39b1bcab2759400284c2fd359d0c4593, entries=150, sequenceid=219, filesize=11.9 K 2024-11-25T05:55:01,603 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:55:01,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T05:55:01,607 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:55:01,608 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for cf8ce3f4f3ad5ac4b5297b0ca903cd34 in 601ms, sequenceid=219, compaction requested=false 2024-11-25T05:55:01,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:01,608 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:55:01,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:55:01,698 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-25T05:55:01,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=A 2024-11-25T05:55:01,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:01,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=B 2024-11-25T05:55:01,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:01,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=C 2024-11-25T05:55:01,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:01,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/febe19c0d4774f8aa30060125b68decf is 50, key is test_row_0/A:col10/1732514101696/Put/seqid=0 2024-11-25T05:55:01,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T05:55:01,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:01,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514161721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:01,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:01,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:01,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514161722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:01,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:01,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:01,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514161722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:01,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:01,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741882_1058 (size=12151) 2024-11-25T05:55:01,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:01,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514161726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:01,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:01,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:01,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/febe19c0d4774f8aa30060125b68decf 2024-11-25T05:55:01,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514161726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:01,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:01,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/7e40c6c31b7745b6acefd3afda962686 is 50, key is test_row_0/B:col10/1732514101696/Put/seqid=0 2024-11-25T05:55:01,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-25T05:55:01,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:01,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:01,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:01,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:01,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:01,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:01,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741883_1059 (size=12151) 2024-11-25T05:55:01,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/7e40c6c31b7745b6acefd3afda962686 2024-11-25T05:55:01,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/ca5604e9287540e198edd05fd0bef9f3 is 50, key is test_row_0/C:col10/1732514101696/Put/seqid=0 2024-11-25T05:55:01,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:01,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514161843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:01,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:01,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514161843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:01,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:01,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:01,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514161843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:01,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:01,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:01,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514161843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:01,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741884_1060 (size=12151) 2024-11-25T05:55:01,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-25T05:55:01,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:01,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:01,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:01,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:01,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:01,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:01,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T05:55:02,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:02,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514162062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:02,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:02,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:02,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:02,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:02,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514162063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:02,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:02,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:02,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:02,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:02,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514162063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:02,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:02,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:02,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:02,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-25T05:55:02,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:02,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:02,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:02,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:02,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:02,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:02,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514162063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:02,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:02,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:02,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:02,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:02,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-25T05:55:02,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:02,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:02,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:02,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:02,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:02,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:02,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T05:55:02,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:02,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514162242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:02,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:02,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:02,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:02,273 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/ca5604e9287540e198edd05fd0bef9f3 2024-11-25T05:55:02,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/febe19c0d4774f8aa30060125b68decf as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/febe19c0d4774f8aa30060125b68decf 2024-11-25T05:55:02,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/febe19c0d4774f8aa30060125b68decf, entries=150, sequenceid=241, filesize=11.9 K 2024-11-25T05:55:02,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/7e40c6c31b7745b6acefd3afda962686 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/7e40c6c31b7745b6acefd3afda962686 2024-11-25T05:55:02,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/7e40c6c31b7745b6acefd3afda962686, entries=150, sequenceid=241, filesize=11.9 K 2024-11-25T05:55:02,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/ca5604e9287540e198edd05fd0bef9f3 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/ca5604e9287540e198edd05fd0bef9f3 2024-11-25T05:55:02,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/ca5604e9287540e198edd05fd0bef9f3, entries=150, sequenceid=241, filesize=11.9 K 2024-11-25T05:55:02,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for cf8ce3f4f3ad5ac4b5297b0ca903cd34 in 633ms, sequenceid=241, compaction requested=true 2024-11-25T05:55:02,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:02,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:55:02,331 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:02,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:02,331 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:02,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:55:02,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:02,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:55:02,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:02,333 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:02,333 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/B is initiating minor compaction (all files) 2024-11-25T05:55:02,333 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/B in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:02,334 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/bb3ccdee8b344b43bd6826ad248f0702, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/d851c78e5e4f40cd89bc3a3d5e0a4f3a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/7e40c6c31b7745b6acefd3afda962686] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=36.0 K 2024-11-25T05:55:02,334 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:02,334 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/A is initiating minor compaction (all files) 2024-11-25T05:55:02,334 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/A in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:02,335 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/4849b1b74f084ae1aa99fbd605ca2b02, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/a3c1a33264dc4388be036a7ecd7546a6, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/febe19c0d4774f8aa30060125b68decf] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=36.0 K 2024-11-25T05:55:02,335 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4849b1b74f084ae1aa99fbd605ca2b02, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732514100250 2024-11-25T05:55:02,336 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting bb3ccdee8b344b43bd6826ad248f0702, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732514100250 2024-11-25T05:55:02,336 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3c1a33264dc4388be036a7ecd7546a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732514100650 2024-11-25T05:55:02,336 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting d851c78e5e4f40cd89bc3a3d5e0a4f3a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732514100650 2024-11-25T05:55:02,336 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting febe19c0d4774f8aa30060125b68decf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732514101050 2024-11-25T05:55:02,337 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e40c6c31b7745b6acefd3afda962686, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732514101050 2024-11-25T05:55:02,355 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#B#compaction#48 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:02,356 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#A#compaction#49 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:02,356 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/bbba73e8e4fa47a3b822dd3b0605b90d is 50, key is test_row_0/B:col10/1732514101696/Put/seqid=0 2024-11-25T05:55:02,356 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/06064228fdbe487d87a70c3b4cebbed2 is 50, key is test_row_0/A:col10/1732514101696/Put/seqid=0 2024-11-25T05:55:02,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741885_1061 (size=12697) 2024-11-25T05:55:02,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:55:02,386 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-25T05:55:02,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-25T05:55:02,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=A 2024-11-25T05:55:02,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:02,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=B 2024-11-25T05:55:02,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:02,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:02,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:02,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=C 2024-11-25T05:55:02,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:02,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:02,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:02,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:02,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:02,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741886_1062 (size=12697) 2024-11-25T05:55:02,392 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/06064228fdbe487d87a70c3b4cebbed2 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/06064228fdbe487d87a70c3b4cebbed2 2024-11-25T05:55:02,398 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/bbba73e8e4fa47a3b822dd3b0605b90d as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/bbba73e8e4fa47a3b822dd3b0605b90d 2024-11-25T05:55:02,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/d79f9f0eafa940b984d7165c3c626cad is 50, key is test_row_0/A:col10/1732514102382/Put/seqid=0 2024-11-25T05:55:02,407 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/B of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into bbba73e8e4fa47a3b822dd3b0605b90d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:02,407 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:02,407 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/B, priority=13, startTime=1732514102331; duration=0sec 2024-11-25T05:55:02,408 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:02,408 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:B 2024-11-25T05:55:02,408 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:02,410 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:02,410 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/C is initiating minor compaction (all files) 2024-11-25T05:55:02,410 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/C in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:02,411 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/A of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into 06064228fdbe487d87a70c3b4cebbed2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:02,411 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:02,411 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/A, priority=13, startTime=1732514102331; duration=0sec 2024-11-25T05:55:02,412 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:02,412 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:A 2024-11-25T05:55:02,414 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/1447f666f6de4f2d84584b89ae5181ad, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/39b1bcab2759400284c2fd359d0c4593, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/ca5604e9287540e198edd05fd0bef9f3] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=36.0 K 2024-11-25T05:55:02,415 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 1447f666f6de4f2d84584b89ae5181ad, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732514100250 2024-11-25T05:55:02,416 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 39b1bcab2759400284c2fd359d0c4593, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732514100650 2024-11-25T05:55:02,417 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting ca5604e9287540e198edd05fd0bef9f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732514101050 2024-11-25T05:55:02,436 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#C#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:02,437 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/f9d0cce73503405aab2c12032ca7207e is 50, key is test_row_0/C:col10/1732514101696/Put/seqid=0 2024-11-25T05:55:02,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:02,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514162431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:02,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:02,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514162433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:02,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:02,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:02,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:02,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:02,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:02,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:02,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:02,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514162443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:02,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:02,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:02,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:02,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:02,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741887_1063 (size=14541) 2024-11-25T05:55:02,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514162443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:02,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:02,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:02,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:02,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741888_1064 (size=12697) 2024-11-25T05:55:02,471 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/f9d0cce73503405aab2c12032ca7207e as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/f9d0cce73503405aab2c12032ca7207e 2024-11-25T05:55:02,485 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/C of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into f9d0cce73503405aab2c12032ca7207e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:02,486 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:02,486 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/C, priority=13, startTime=1732514102332; duration=0sec 2024-11-25T05:55:02,486 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:02,486 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:C 2024-11-25T05:55:02,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-25T05:55:02,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:02,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:02,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:02,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:02,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:02,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:02,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:02,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514162552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:02,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:02,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:02,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:02,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:02,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:02,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514162552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:02,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514162552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:02,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:02,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:02,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:02,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:02,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:02,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:02,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:02,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514162553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:02,558 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:02,558 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:02,558 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:02,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-25T05:55:02,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:02,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:02,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:02,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:02,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:02,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:02,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T05:55:02,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:02,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514162762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:02,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:02,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:02,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:02,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:02,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514162763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:02,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:02,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:02,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:02,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:02,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514162763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:02,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:02,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514162763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:02,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:02,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:02,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:02,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:02,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:02,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:02,844 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/d79f9f0eafa940b984d7165c3c626cad 2024-11-25T05:55:02,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-25T05:55:02,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:02,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:02,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:02,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:02,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:02,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:02,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/e8f22d60223443cab44f3e15b88a5e15 is 50, key is test_row_0/B:col10/1732514102382/Put/seqid=0 2024-11-25T05:55:02,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741889_1065 (size=12151) 2024-11-25T05:55:02,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/e8f22d60223443cab44f3e15b88a5e15 2024-11-25T05:55:02,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/9ee0e1b62776466bacc959514a3f2b6b is 50, key is test_row_0/C:col10/1732514102382/Put/seqid=0 2024-11-25T05:55:02,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741890_1066 (size=12151) 2024-11-25T05:55:02,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/9ee0e1b62776466bacc959514a3f2b6b 2024-11-25T05:55:02,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/d79f9f0eafa940b984d7165c3c626cad as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/d79f9f0eafa940b984d7165c3c626cad 2024-11-25T05:55:02,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/d79f9f0eafa940b984d7165c3c626cad, entries=200, sequenceid=256, filesize=14.2 K 2024-11-25T05:55:02,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/e8f22d60223443cab44f3e15b88a5e15 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/e8f22d60223443cab44f3e15b88a5e15 2024-11-25T05:55:02,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/e8f22d60223443cab44f3e15b88a5e15, entries=150, sequenceid=256, filesize=11.9 K 2024-11-25T05:55:02,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/9ee0e1b62776466bacc959514a3f2b6b as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/9ee0e1b62776466bacc959514a3f2b6b 2024-11-25T05:55:03,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/9ee0e1b62776466bacc959514a3f2b6b, entries=150, sequenceid=256, filesize=11.9 K 2024-11-25T05:55:03,007 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for cf8ce3f4f3ad5ac4b5297b0ca903cd34 in 621ms, sequenceid=256, compaction requested=false 2024-11-25T05:55:03,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:03,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-25T05:55:03,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:03,019 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-25T05:55:03,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=A 2024-11-25T05:55:03,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:03,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=B 2024-11-25T05:55:03,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:03,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=C 2024-11-25T05:55:03,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:03,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/4a07ef751a6141fc8681ca0836caa6e2 is 50, key is test_row_0/A:col10/1732514102437/Put/seqid=0 2024-11-25T05:55:03,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741891_1067 (size=12301) 2024-11-25T05:55:03,054 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/4a07ef751a6141fc8681ca0836caa6e2 2024-11-25T05:55:03,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/85c3b3d45b9942648f03a82875290b96 is 50, key is test_row_0/B:col10/1732514102437/Put/seqid=0 2024-11-25T05:55:03,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:55:03,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:03,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741892_1068 (size=12301) 2024-11-25T05:55:03,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:03,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514163110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:03,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:03,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514163110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:03,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:03,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:03,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514163111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:03,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:03,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:03,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514163112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:03,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:03,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:03,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:03,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514163232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:03,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:03,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:03,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514163233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:03,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:03,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514163233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:03,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:03,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514163233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:03,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:03,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:03,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514163262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:03,271 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_2' in cf8ce3f4f3ad5ac4b5297b0ca903cd34 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:55:03,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:03,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:03,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514163442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:03,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:03,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514163442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:03,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:03,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514163442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:03,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:03,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:03,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:03,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:03,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514163453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:03,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:03,509 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/85c3b3d45b9942648f03a82875290b96 2024-11-25T05:55:03,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/c9030d10f7bb4138a21480061df66bb5 is 50, key is test_row_0/C:col10/1732514102437/Put/seqid=0 2024-11-25T05:55:03,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741893_1069 (size=12301) 2024-11-25T05:55:03,545 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/c9030d10f7bb4138a21480061df66bb5 2024-11-25T05:55:03,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/4a07ef751a6141fc8681ca0836caa6e2 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/4a07ef751a6141fc8681ca0836caa6e2 2024-11-25T05:55:03,562 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/4a07ef751a6141fc8681ca0836caa6e2, entries=150, sequenceid=277, filesize=12.0 K 2024-11-25T05:55:03,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/85c3b3d45b9942648f03a82875290b96 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/85c3b3d45b9942648f03a82875290b96 2024-11-25T05:55:03,575 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/85c3b3d45b9942648f03a82875290b96, entries=150, sequenceid=277, filesize=12.0 K 2024-11-25T05:55:03,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/c9030d10f7bb4138a21480061df66bb5 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/c9030d10f7bb4138a21480061df66bb5 2024-11-25T05:55:03,591 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/c9030d10f7bb4138a21480061df66bb5, entries=150, sequenceid=277, filesize=12.0 K 2024-11-25T05:55:03,593 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for cf8ce3f4f3ad5ac4b5297b0ca903cd34 in 574ms, sequenceid=277, compaction requested=true 2024-11-25T05:55:03,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:03,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:03,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-25T05:55:03,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-25T05:55:03,599 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-25T05:55:03,599 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9870 sec 2024-11-25T05:55:03,603 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.0070 sec 2024-11-25T05:55:03,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:55:03,758 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-25T05:55:03,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=A 2024-11-25T05:55:03,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:03,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=B 2024-11-25T05:55:03,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:03,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=C 2024-11-25T05:55:03,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:03,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T05:55:03,762 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:55:03,764 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:55:03,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-25T05:55:03,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-25T05:55:03,767 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:55:03,768 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:55:03,769 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:55:03,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/1883ff8e3bbc439ebc31c74e304527f2 is 50, key is test_row_0/A:col10/1732514103753/Put/seqid=0 2024-11-25T05:55:03,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741894_1070 (size=14741) 2024-11-25T05:55:03,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/1883ff8e3bbc439ebc31c74e304527f2 2024-11-25T05:55:03,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/e84ab1ad464545aa853d71d7456949dd is 50, key is test_row_0/B:col10/1732514103753/Put/seqid=0 2024-11-25T05:55:03,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741895_1071 (size=12301) 2024-11-25T05:55:03,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:03,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514163805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:03,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:03,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514163797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:03,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:03,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:03,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514163812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:03,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:03,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:03,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:03,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514163812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:03,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:03,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-25T05:55:03,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:03,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514163922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:03,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:03,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:03,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514163923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:03,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:03,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514163923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:03,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:03,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:03,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514163923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:03,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:03,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-25T05:55:03,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:03,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:03,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:03,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:03,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:03,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:03,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:03,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:04,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-25T05:55:04,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-25T05:55:04,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:04,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:04,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:04,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:04,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:04,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:04,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:04,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514164132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:04,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:04,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:04,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:04,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:04,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514164132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:04,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:04,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:04,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:04,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:04,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514164132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:04,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:04,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:04,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:04,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:04,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514164132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:04,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:04,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:04,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:04,212 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/e84ab1ad464545aa853d71d7456949dd 2024-11-25T05:55:04,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/9682072354ec4edaa97962198b6318f5 is 50, key is test_row_0/C:col10/1732514103753/Put/seqid=0 2024-11-25T05:55:04,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-25T05:55:04,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:04,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:04,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:04,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:04,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:04,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:04,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741896_1072 (size=12301) 2024-11-25T05:55:04,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/9682072354ec4edaa97962198b6318f5 2024-11-25T05:55:04,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/1883ff8e3bbc439ebc31c74e304527f2 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/1883ff8e3bbc439ebc31c74e304527f2 2024-11-25T05:55:04,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/1883ff8e3bbc439ebc31c74e304527f2, entries=200, sequenceid=292, filesize=14.4 K 2024-11-25T05:55:04,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/e84ab1ad464545aa853d71d7456949dd as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/e84ab1ad464545aa853d71d7456949dd 2024-11-25T05:55:04,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/e84ab1ad464545aa853d71d7456949dd, entries=150, sequenceid=292, filesize=12.0 K 2024-11-25T05:55:04,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/9682072354ec4edaa97962198b6318f5 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/9682072354ec4edaa97962198b6318f5 2024-11-25T05:55:04,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/9682072354ec4edaa97962198b6318f5, entries=150, sequenceid=292, filesize=12.0 K 2024-11-25T05:55:04,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for cf8ce3f4f3ad5ac4b5297b0ca903cd34 in 554ms, sequenceid=292, compaction requested=true 2024-11-25T05:55:04,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:04,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:55:04,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:04,312 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T05:55:04,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:55:04,312 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T05:55:04,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:04,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:55:04,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:04,314 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T05:55:04,314 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/B is initiating minor compaction (all files) 2024-11-25T05:55:04,314 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/B in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:04,315 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/bbba73e8e4fa47a3b822dd3b0605b90d, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/e8f22d60223443cab44f3e15b88a5e15, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/85c3b3d45b9942648f03a82875290b96, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/e84ab1ad464545aa853d71d7456949dd] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=48.3 K 2024-11-25T05:55:04,315 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54280 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T05:55:04,315 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/A is initiating minor compaction (all files) 2024-11-25T05:55:04,315 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/A in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:04,316 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting bbba73e8e4fa47a3b822dd3b0605b90d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732514101050 2024-11-25T05:55:04,316 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/06064228fdbe487d87a70c3b4cebbed2, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/d79f9f0eafa940b984d7165c3c626cad, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/4a07ef751a6141fc8681ca0836caa6e2, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/1883ff8e3bbc439ebc31c74e304527f2] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=53.0 K 2024-11-25T05:55:04,316 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting e8f22d60223443cab44f3e15b88a5e15, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732514101718 2024-11-25T05:55:04,316 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06064228fdbe487d87a70c3b4cebbed2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732514101050 2024-11-25T05:55:04,317 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 85c3b3d45b9942648f03a82875290b96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732514102430 2024-11-25T05:55:04,317 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting d79f9f0eafa940b984d7165c3c626cad, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732514101718 2024-11-25T05:55:04,317 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting e84ab1ad464545aa853d71d7456949dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732514103076 2024-11-25T05:55:04,317 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a07ef751a6141fc8681ca0836caa6e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732514102430 2024-11-25T05:55:04,324 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1883ff8e3bbc439ebc31c74e304527f2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732514103076 2024-11-25T05:55:04,336 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#B#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:04,337 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/4bea555fd78e4741b78b05b18032d5f3 is 50, key is test_row_0/B:col10/1732514103753/Put/seqid=0 2024-11-25T05:55:04,343 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#A#compaction#61 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:04,344 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/d02c8391e19d408b99053bfece220ba8 is 50, key is test_row_0/A:col10/1732514103753/Put/seqid=0 2024-11-25T05:55:04,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741897_1073 (size=12983) 2024-11-25T05:55:04,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741898_1074 (size=12983) 2024-11-25T05:55:04,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-25T05:55:04,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:04,392 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-25T05:55:04,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-25T05:55:04,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=A 2024-11-25T05:55:04,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:04,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=B 2024-11-25T05:55:04,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:04,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=C 2024-11-25T05:55:04,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:04,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/019b2688c576421eb675d3f83d6da1b0 is 50, key is test_row_0/A:col10/1732514103809/Put/seqid=0 2024-11-25T05:55:04,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741899_1075 (size=12301) 2024-11-25T05:55:04,426 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/019b2688c576421eb675d3f83d6da1b0 2024-11-25T05:55:04,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:55:04,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:04,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/bccf2a3721eb4512a0ed3be82bd208e7 is 50, key is test_row_0/B:col10/1732514103809/Put/seqid=0 2024-11-25T05:55:04,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741900_1076 (size=12301) 2024-11-25T05:55:04,471 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/bccf2a3721eb4512a0ed3be82bd208e7 2024-11-25T05:55:04,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:04,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514164485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:04,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:04,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:04,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:04,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:04,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514164488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:04,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:04,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:04,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:04,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:04,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514164489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:04,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:04,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:04,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514164490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:04,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:04,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:04,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:04,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:04,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:04,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/901be871432244649e76501a29e2e3ba is 50, key is test_row_0/C:col10/1732514103809/Put/seqid=0 2024-11-25T05:55:04,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741901_1077 (size=12301) 2024-11-25T05:55:04,544 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/901be871432244649e76501a29e2e3ba 2024-11-25T05:55:04,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/019b2688c576421eb675d3f83d6da1b0 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/019b2688c576421eb675d3f83d6da1b0 2024-11-25T05:55:04,571 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/019b2688c576421eb675d3f83d6da1b0, entries=150, sequenceid=310, filesize=12.0 K 2024-11-25T05:55:04,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/bccf2a3721eb4512a0ed3be82bd208e7 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/bccf2a3721eb4512a0ed3be82bd208e7 2024-11-25T05:55:04,587 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/bccf2a3721eb4512a0ed3be82bd208e7, entries=150, sequenceid=310, filesize=12.0 K 2024-11-25T05:55:04,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/901be871432244649e76501a29e2e3ba as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/901be871432244649e76501a29e2e3ba 2024-11-25T05:55:04,601 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/901be871432244649e76501a29e2e3ba, entries=150, sequenceid=310, filesize=12.0 K 2024-11-25T05:55:04,606 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for cf8ce3f4f3ad5ac4b5297b0ca903cd34 in 211ms, sequenceid=310, compaction requested=true 2024-11-25T05:55:04,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:04,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:04,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-25T05:55:04,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-25T05:55:04,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:55:04,619 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-25T05:55:04,624 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-25T05:55:04,625 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 849 msec 2024-11-25T05:55:04,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=A 2024-11-25T05:55:04,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:04,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=B 2024-11-25T05:55:04,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:04,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=C 2024-11-25T05:55:04,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:04,628 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 862 msec 2024-11-25T05:55:04,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/cd77d6986b124564914e8449cace7a8d is 50, key is test_row_0/A:col10/1732514104615/Put/seqid=0 2024-11-25T05:55:04,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:04,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514164657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:04,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:04,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514164655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:04,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:04,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:04,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:04,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:04,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:04,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:04,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:04,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514164673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:04,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741902_1078 (size=17181) 2024-11-25T05:55:04,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:04,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:04,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:04,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:04,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514164673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:04,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:04,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:04,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:04,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/cd77d6986b124564914e8449cace7a8d 2024-11-25T05:55:04,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/d13a2e69eb2e4a4da160228f4a95ccca is 50, key is test_row_0/B:col10/1732514104615/Put/seqid=0 2024-11-25T05:55:04,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741903_1079 (size=12301) 2024-11-25T05:55:04,764 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/d02c8391e19d408b99053bfece220ba8 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/d02c8391e19d408b99053bfece220ba8 2024-11-25T05:55:04,764 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/4bea555fd78e4741b78b05b18032d5f3 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/4bea555fd78e4741b78b05b18032d5f3 2024-11-25T05:55:04,776 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/B of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into 4bea555fd78e4741b78b05b18032d5f3(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:04,776 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:04,776 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/B, priority=12, startTime=1732514104312; duration=0sec 2024-11-25T05:55:04,776 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:04,776 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:B 2024-11-25T05:55:04,776 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-25T05:55:04,780 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61751 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-25T05:55:04,780 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/C is initiating minor compaction (all files) 2024-11-25T05:55:04,780 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/C in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:04,781 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/f9d0cce73503405aab2c12032ca7207e, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/9ee0e1b62776466bacc959514a3f2b6b, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/c9030d10f7bb4138a21480061df66bb5, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/9682072354ec4edaa97962198b6318f5, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/901be871432244649e76501a29e2e3ba] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=60.3 K 2024-11-25T05:55:04,781 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting f9d0cce73503405aab2c12032ca7207e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732514101050 2024-11-25T05:55:04,782 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ee0e1b62776466bacc959514a3f2b6b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732514101718 2024-11-25T05:55:04,783 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting c9030d10f7bb4138a21480061df66bb5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732514102430 2024-11-25T05:55:04,784 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 9682072354ec4edaa97962198b6318f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732514103076 2024-11-25T05:55:04,785 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 901be871432244649e76501a29e2e3ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732514103783 2024-11-25T05:55:04,787 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/A of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into d02c8391e19d408b99053bfece220ba8(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:04,787 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:04,787 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/A, priority=12, startTime=1732514104311; duration=0sec 2024-11-25T05:55:04,787 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:04,787 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:A 2024-11-25T05:55:04,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:04,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514164782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:04,798 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:04,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:04,798 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:04,798 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:04,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514164782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:04,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:04,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514164782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:04,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:04,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514164793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:04,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:04,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:04,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:04,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:04,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:04,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:04,803 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#C#compaction#67 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:04,803 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/a9f43efffc134fefb943202038188b19 is 50, key is test_row_0/C:col10/1732514103809/Put/seqid=0 2024-11-25T05:55:04,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:04,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:04,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:04,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741904_1080 (size=13017) 2024-11-25T05:55:04,839 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/a9f43efffc134fefb943202038188b19 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/a9f43efffc134fefb943202038188b19 2024-11-25T05:55:04,856 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/C of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into a9f43efffc134fefb943202038188b19(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:04,856 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:04,856 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/C, priority=11, startTime=1732514104312; duration=0sec 2024-11-25T05:55:04,857 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:04,857 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:C 2024-11-25T05:55:04,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-25T05:55:04,903 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:55:04,903 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:55:04,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-25T05:55:04,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T05:55:04,906 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:55:04,907 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:55:04,907 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:55:05,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:05,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514165002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:05,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:05,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:05,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:05,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514165002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:05,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514165002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:05,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T05:55:05,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:05,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514165013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:05,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:05,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T05:55:05,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:05,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:05,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:05,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:05,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:05,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:05,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/d13a2e69eb2e4a4da160228f4a95ccca 2024-11-25T05:55:05,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/06a24c6f5e554b4a86a91652936cdac9 is 50, key is test_row_0/C:col10/1732514104615/Put/seqid=0 2024-11-25T05:55:05,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741905_1081 (size=12301) 2024-11-25T05:55:05,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/06a24c6f5e554b4a86a91652936cdac9 2024-11-25T05:55:05,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/cd77d6986b124564914e8449cace7a8d as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/cd77d6986b124564914e8449cace7a8d 2024-11-25T05:55:05,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/cd77d6986b124564914e8449cace7a8d, entries=250, sequenceid=328, filesize=16.8 K 2024-11-25T05:55:05,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/d13a2e69eb2e4a4da160228f4a95ccca as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/d13a2e69eb2e4a4da160228f4a95ccca 2024-11-25T05:55:05,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/d13a2e69eb2e4a4da160228f4a95ccca, entries=150, sequenceid=328, filesize=12.0 K 2024-11-25T05:55:05,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/06a24c6f5e554b4a86a91652936cdac9 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/06a24c6f5e554b4a86a91652936cdac9 2024-11-25T05:55:05,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T05:55:05,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:05,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:05,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:05,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:05,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:05,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T05:55:05,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/06a24c6f5e554b4a86a91652936cdac9, entries=150, sequenceid=328, filesize=12.0 K 2024-11-25T05:55:05,229 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for cf8ce3f4f3ad5ac4b5297b0ca903cd34 in 610ms, sequenceid=328, compaction requested=true 2024-11-25T05:55:05,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:05,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:55:05,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:05,230 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:05,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:55:05,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:05,230 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:05,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:55:05,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:05,232 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42465 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:05,232 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:05,232 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/A is initiating minor compaction (all files) 2024-11-25T05:55:05,232 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/B is initiating minor compaction (all files) 2024-11-25T05:55:05,232 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/A in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:05,232 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/B in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:05,232 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/d02c8391e19d408b99053bfece220ba8, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/019b2688c576421eb675d3f83d6da1b0, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/cd77d6986b124564914e8449cace7a8d] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=41.5 K 2024-11-25T05:55:05,232 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/4bea555fd78e4741b78b05b18032d5f3, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/bccf2a3721eb4512a0ed3be82bd208e7, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/d13a2e69eb2e4a4da160228f4a95ccca] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=36.7 K 2024-11-25T05:55:05,234 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bea555fd78e4741b78b05b18032d5f3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732514103076 2024-11-25T05:55:05,234 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting d02c8391e19d408b99053bfece220ba8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732514103076 2024-11-25T05:55:05,234 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting bccf2a3721eb4512a0ed3be82bd208e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732514103783 2024-11-25T05:55:05,235 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting d13a2e69eb2e4a4da160228f4a95ccca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732514104609 2024-11-25T05:55:05,235 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 019b2688c576421eb675d3f83d6da1b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732514103783 2024-11-25T05:55:05,235 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd77d6986b124564914e8449cace7a8d, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732514104484 2024-11-25T05:55:05,263 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#B#compaction#69 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:05,264 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/5c1c949d87cb4ea0821f1de093be05e8 is 50, key is test_row_0/B:col10/1732514104615/Put/seqid=0 2024-11-25T05:55:05,264 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#A#compaction#70 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:05,265 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/2086a1af63c34adca14f3c3230bd9050 is 50, key is test_row_0/A:col10/1732514104615/Put/seqid=0 2024-11-25T05:55:05,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741907_1083 (size=13085) 2024-11-25T05:55:05,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:55:05,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-25T05:55:05,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=A 2024-11-25T05:55:05,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:05,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=B 2024-11-25T05:55:05,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:05,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=C 2024-11-25T05:55:05,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:05,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741906_1082 (size=13085) 2024-11-25T05:55:05,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/20da8daddde54b578bcbc1194295d682 is 50, key is test_row_0/A:col10/1732514104653/Put/seqid=0 2024-11-25T05:55:05,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741908_1084 (size=14741) 2024-11-25T05:55:05,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/20da8daddde54b578bcbc1194295d682 2024-11-25T05:55:05,325 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/5c1c949d87cb4ea0821f1de093be05e8 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/5c1c949d87cb4ea0821f1de093be05e8 2024-11-25T05:55:05,335 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/B of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into 5c1c949d87cb4ea0821f1de093be05e8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:05,335 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:05,335 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/B, priority=13, startTime=1732514105230; duration=0sec 2024-11-25T05:55:05,335 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:05,335 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:B 2024-11-25T05:55:05,335 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-25T05:55:05,336 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-25T05:55:05,336 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-25T05:55:05,336 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. because compaction request was cancelled 2024-11-25T05:55:05,336 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:C 2024-11-25T05:55:05,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/7ed88332848b41159f064f0874b03115 is 50, key is test_row_0/B:col10/1732514104653/Put/seqid=0 2024-11-25T05:55:05,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741909_1085 (size=12301) 2024-11-25T05:55:05,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/7ed88332848b41159f064f0874b03115 2024-11-25T05:55:05,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:05,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514165345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:05,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:05,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:05,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:05,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514165360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:05,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514165356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:05,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:05,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:05,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:05,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514165361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:05,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:05,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:05,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514165364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:05,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:05,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T05:55:05,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/642c20ea3e7e4a69b7641e4630b8912a is 50, key is test_row_0/C:col10/1732514104653/Put/seqid=0 2024-11-25T05:55:05,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:05,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:05,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:05,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:05,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:05,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:05,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741910_1086 (size=12301) 2024-11-25T05:55:05,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/642c20ea3e7e4a69b7641e4630b8912a 2024-11-25T05:55:05,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/20da8daddde54b578bcbc1194295d682 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/20da8daddde54b578bcbc1194295d682 2024-11-25T05:55:05,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/20da8daddde54b578bcbc1194295d682, entries=200, sequenceid=347, filesize=14.4 K 2024-11-25T05:55:05,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/7ed88332848b41159f064f0874b03115 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/7ed88332848b41159f064f0874b03115 2024-11-25T05:55:05,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/7ed88332848b41159f064f0874b03115, entries=150, sequenceid=347, filesize=12.0 K 2024-11-25T05:55:05,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/642c20ea3e7e4a69b7641e4630b8912a as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/642c20ea3e7e4a69b7641e4630b8912a 2024-11-25T05:55:05,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/642c20ea3e7e4a69b7641e4630b8912a, entries=150, sequenceid=347, filesize=12.0 K 2024-11-25T05:55:05,437 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=100.63 KB/103050 for cf8ce3f4f3ad5ac4b5297b0ca903cd34 in 138ms, sequenceid=347, compaction requested=true 2024-11-25T05:55:05,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:05,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:55:05,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:05,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:55:05,437 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-11-25T05:55:05,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:05,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:55:05,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T05:55:05,438 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-25T05:55:05,438 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-25T05:55:05,438 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. because compaction request was cancelled 2024-11-25T05:55:05,438 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:A 2024-11-25T05:55:05,438 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:05,439 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:05,439 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/C is initiating minor compaction (all files) 2024-11-25T05:55:05,439 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/C in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:05,440 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/a9f43efffc134fefb943202038188b19, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/06a24c6f5e554b4a86a91652936cdac9, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/642c20ea3e7e4a69b7641e4630b8912a] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=36.7 K 2024-11-25T05:55:05,441 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting a9f43efffc134fefb943202038188b19, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732514103783 2024-11-25T05:55:05,441 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 06a24c6f5e554b4a86a91652936cdac9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732514104609 2024-11-25T05:55:05,442 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 642c20ea3e7e4a69b7641e4630b8912a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732514104651 2024-11-25T05:55:05,455 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#C#compaction#74 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:05,456 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/d4b15ed789c440ab9c9c46ee3cc6bfbe is 50, key is test_row_0/C:col10/1732514104653/Put/seqid=0 2024-11-25T05:55:05,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741911_1087 (size=13119) 2024-11-25T05:55:05,485 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-25T05:55:05,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:55:05,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732514105357, max timestamp=1732514105485Num uniques 859; 2024-11-25T05:55:05,492 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T05:55:05,492 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732514105357, max timestamp=1732514105485Num uniques 859; 2024-11-25T05:55:05,492 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T05:55:05,494 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732514105357, max timestamp=1732514105485Num uniques 859; 2024-11-25T05:55:05,494 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T05:55:05,501 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/d4b15ed789c440ab9c9c46ee3cc6bfbe as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/d4b15ed789c440ab9c9c46ee3cc6bfbe 2024-11-25T05:55:05,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=A 2024-11-25T05:55:05,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:55:05,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=B 2024-11-25T05:55:05,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:55:05,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=C 2024-11-25T05:55:05,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:55:05,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/61b5b9fb1fa44761a83a4b25ddd3afd8 is 50, key is test_row_0/A:col10/1732514105486/Put/seqid=0 2024-11-25T05:55:05,523 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/C of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into d4b15ed789c440ab9c9c46ee3cc6bfbe(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:05,524 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:05,524 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/C, priority=13, startTime=1732514105437; duration=0sec 2024-11-25T05:55:05,524 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:05,525 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:C 2024-11-25T05:55:05,526 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-25T05:55:05,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T05:55:05,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:05,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:05,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:05,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:05,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:05,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:05,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T05:55:05,533 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-25T05:55:05,533 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-25T05:55:05,533 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. because compaction request was cancelled 2024-11-25T05:55:05,533 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:B 2024-11-25T05:55:05,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741912_1088 (size=12301) 2024-11-25T05:55:05,559 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/61b5b9fb1fa44761a83a4b25ddd3afd8 2024-11-25T05:55:05,574 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732514105544, max timestamp=1732514105570Num uniques 859; 2024-11-25T05:55:05,574 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-25T05:55:05,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732514105552, max timestamp=1732514105570Num uniques 859; 2024-11-25T05:55:05,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T05:55:05,580 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732514105552, max timestamp=1732514105570Num uniques 859; 2024-11-25T05:55:05,580 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T05:55:05,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/bdc10dfc947f45d5a0a62bb816989fe7 is 50, key is test_row_0/B:col10/1732514105486/Put/seqid=0 2024-11-25T05:55:05,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741913_1089 (size=12301) 2024-11-25T05:55:05,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1732514105604, max timestamp=1732514105634Num uniques 1168; 2024-11-25T05:55:05,640 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=16848, suffix heap size=149440,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=368 2024-11-25T05:55:05,641 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1732514105604, max timestamp=1732514105634Num uniques 1018; 2024-11-25T05:55:05,641 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=16848, suffix heap size=133840,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=368 2024-11-25T05:55:05,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732514105616, max timestamp=1732514105634Num uniques 1018; 2024-11-25T05:55:05,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-25T05:55:05,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T05:55:05,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:05,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:05,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:05,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:05,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:05,695 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732514105675, max timestamp=1732514105690Num uniques 1227; 2024-11-25T05:55:05,695 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=56192, new segment data size=10392, suffix heap size=155576,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1227, new segment cells count=227 2024-11-25T05:55:05,695 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732514105675, max timestamp=1732514105690Num uniques 1227; 2024-11-25T05:55:05,695 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=56192, new segment data size=10392, suffix heap size=155576,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1227, new segment cells count=227 2024-11-25T05:55:05,699 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732514105675, max timestamp=1732514105690Num uniques 1077; 2024-11-25T05:55:05,699 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-25T05:55:05,702 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/2086a1af63c34adca14f3c3230bd9050 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/2086a1af63c34adca14f3c3230bd9050 2024-11-25T05:55:05,717 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/A of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into 2086a1af63c34adca14f3c3230bd9050(size=12.8 K), total size for store is 27.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:05,717 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:05,717 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/A, priority=13, startTime=1732514105230; duration=0sec 2024-11-25T05:55:05,717 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:05,717 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:A 2024-11-25T05:55:05,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=536, cellSize=24546, totalHeapSize=55984, min timestamp=1732514105709, max timestamp=1732514105736Num uniques 1086; 2024-11-25T05:55:05,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=24546, suffix heap size=140912,new segment heap size=55984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=536 2024-11-25T05:55:05,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=536, cellSize=24546, totalHeapSize=55984, min timestamp=1732514105709, max timestamp=1732514105736Num uniques 1086; 2024-11-25T05:55:05,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=24546, suffix heap size=140912,new segment heap size=55984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=536 2024-11-25T05:55:05,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=536, cellSize=24546, totalHeapSize=55984, min timestamp=1732514105709, max timestamp=1732514105736Num uniques 1086; 2024-11-25T05:55:05,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=24546, suffix heap size=140912,new segment heap size=55984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=536 2024-11-25T05:55:05,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:05,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514165777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:05,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:05,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514165778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:05,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:05,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:05,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:05,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514165779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:05,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:05,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:05,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514165785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:05,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:05,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:05,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514165788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:05,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:05,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T05:55:05,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:05,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:05,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:05,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:05,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:05,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:05,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:05,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514165892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:05,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:05,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514165892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:05,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:05,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:05,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:05,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:05,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514165902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:05,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514165902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:05,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:05,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514165902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:05,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:05,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:05,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:05,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:05,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T05:55:05,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:05,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:05,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:05,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:05,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:06,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/bdc10dfc947f45d5a0a62bb816989fe7 2024-11-25T05:55:06,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T05:55:06,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/81bde786ad2c435da00aee08ac9b5b52 is 50, key is test_row_0/C:col10/1732514105486/Put/seqid=0 2024-11-25T05:55:06,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741914_1090 (size=12301) 2024-11-25T05:55:06,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514166102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:06,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:06,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:06,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:06,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:06,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514166106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:06,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:06,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:06,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:06,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514166122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:06,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:06,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:06,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514166122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:06,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:06,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:06,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:06,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:06,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514166122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:06,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:06,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:06,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:06,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T05:55:06,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:06,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:06,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:06,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:06,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:06,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T05:55:06,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:06,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:06,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:06,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:06,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:06,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514166412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:06,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:06,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:06,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:06,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514166412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:06,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:06,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:06,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:06,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:06,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514166431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:06,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:06,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:06,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514166432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:06,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:06,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:06,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:06,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:06,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:06,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:06,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514166442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:06,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:06,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:06,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:06,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/81bde786ad2c435da00aee08ac9b5b52 2024-11-25T05:55:06,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/61b5b9fb1fa44761a83a4b25ddd3afd8 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/61b5b9fb1fa44761a83a4b25ddd3afd8 2024-11-25T05:55:06,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/61b5b9fb1fa44761a83a4b25ddd3afd8, entries=150, sequenceid=370, filesize=12.0 K 2024-11-25T05:55:06,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T05:55:06,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/bdc10dfc947f45d5a0a62bb816989fe7 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/bdc10dfc947f45d5a0a62bb816989fe7 2024-11-25T05:55:06,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:06,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:06,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:06,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:06,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:06,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:06,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/bdc10dfc947f45d5a0a62bb816989fe7, entries=150, sequenceid=370, filesize=12.0 K 2024-11-25T05:55:06,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/81bde786ad2c435da00aee08ac9b5b52 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/81bde786ad2c435da00aee08ac9b5b52 2024-11-25T05:55:06,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/81bde786ad2c435da00aee08ac9b5b52, entries=150, sequenceid=370, filesize=12.0 K 2024-11-25T05:55:06,509 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=161.02 KB/164880 for cf8ce3f4f3ad5ac4b5297b0ca903cd34 in 1024ms, sequenceid=370, compaction requested=true 2024-11-25T05:55:06,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:06,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:55:06,510 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:06,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:06,510 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:06,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:55:06,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:06,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:55:06,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:06,512 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:06,512 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/B is initiating minor compaction (all files) 2024-11-25T05:55:06,512 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/B in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:06,512 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/5c1c949d87cb4ea0821f1de093be05e8, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/7ed88332848b41159f064f0874b03115, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/bdc10dfc947f45d5a0a62bb816989fe7] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=36.8 K 2024-11-25T05:55:06,512 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:06,513 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/A is initiating minor compaction (all files) 2024-11-25T05:55:06,513 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/A in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:06,513 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/2086a1af63c34adca14f3c3230bd9050, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/20da8daddde54b578bcbc1194295d682, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/61b5b9fb1fa44761a83a4b25ddd3afd8] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=39.2 K 2024-11-25T05:55:06,513 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2086a1af63c34adca14f3c3230bd9050, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732514104609 2024-11-25T05:55:06,514 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c1c949d87cb4ea0821f1de093be05e8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732514104609 2024-11-25T05:55:06,514 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20da8daddde54b578bcbc1194295d682, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732514104651 2024-11-25T05:55:06,514 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61b5b9fb1fa44761a83a4b25ddd3afd8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732514105357 2024-11-25T05:55:06,515 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ed88332848b41159f064f0874b03115, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732514104651 2024-11-25T05:55:06,518 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting bdc10dfc947f45d5a0a62bb816989fe7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732514105357 2024-11-25T05:55:06,544 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#A#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:06,546 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/80b0a231452446e2b5138a0368f038f7 is 50, key is test_row_0/A:col10/1732514105486/Put/seqid=0 2024-11-25T05:55:06,551 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#B#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:06,551 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/7fe7cae59ee145fdac968a7b9d7767c5 is 50, key is test_row_0/B:col10/1732514105486/Put/seqid=0 2024-11-25T05:55:06,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741915_1091 (size=13187) 2024-11-25T05:55:06,577 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/80b0a231452446e2b5138a0368f038f7 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/80b0a231452446e2b5138a0368f038f7 2024-11-25T05:55:06,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741916_1092 (size=13187) 2024-11-25T05:55:06,589 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/A of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into 80b0a231452446e2b5138a0368f038f7(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:06,589 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:06,590 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/A, priority=13, startTime=1732514106509; duration=0sec 2024-11-25T05:55:06,591 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:06,591 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:A 2024-11-25T05:55:06,592 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/7fe7cae59ee145fdac968a7b9d7767c5 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/7fe7cae59ee145fdac968a7b9d7767c5 2024-11-25T05:55:06,592 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-25T05:55:06,594 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-25T05:55:06,594 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-25T05:55:06,594 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. because compaction request was cancelled 2024-11-25T05:55:06,594 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:C 2024-11-25T05:55:06,601 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/B of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into 7fe7cae59ee145fdac968a7b9d7767c5(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:06,602 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:06,602 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/B, priority=13, startTime=1732514106510; duration=0sec 2024-11-25T05:55:06,603 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:06,603 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:B 2024-11-25T05:55:06,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T05:55:06,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:06,635 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34 3/3 column families, dataSize=161.02 KB heapSize=429.33 KB 2024-11-25T05:55:06,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=A 2024-11-25T05:55:06,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:55:06,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=B 2024-11-25T05:55:06,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:55:06,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=C 2024-11-25T05:55:06,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:55:06,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/c702903f5230430995613720e614581b is 50, key is test_row_0/A:col10/1732514105776/Put/seqid=0 2024-11-25T05:55:06,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741917_1093 (size=12301) 2024-11-25T05:55:06,655 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/c702903f5230430995613720e614581b 2024-11-25T05:55:06,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/072b55ca7e6048f2a95f77f521b0c840 is 50, key is test_row_0/B:col10/1732514105776/Put/seqid=0 2024-11-25T05:55:06,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741918_1094 (size=12301) 2024-11-25T05:55:06,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:55:06,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:06,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:06,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514166953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:06,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:06,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:06,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:06,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514166953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:06,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:06,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:06,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:06,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514166963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:06,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:06,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:06,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:06,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514166965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:06,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:06,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:06,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:06,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:06,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514166970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:06,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:06,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:06,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:07,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T05:55:07,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:07,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514167073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:07,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:07,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:07,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:07,077 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/072b55ca7e6048f2a95f77f521b0c840 2024-11-25T05:55:07,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514167082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:07,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:07,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:07,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:07,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:07,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514167083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:07,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:07,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:07,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:07,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/6a4b3abbaab3403bbd7df5d64bca60e7 is 50, key is test_row_0/C:col10/1732514105776/Put/seqid=0 2024-11-25T05:55:07,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741919_1095 (size=12301) 2024-11-25T05:55:07,151 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/6a4b3abbaab3403bbd7df5d64bca60e7 2024-11-25T05:55:07,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/c702903f5230430995613720e614581b as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/c702903f5230430995613720e614581b 2024-11-25T05:55:07,170 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/c702903f5230430995613720e614581b, entries=150, sequenceid=459, filesize=12.0 K 2024-11-25T05:55:07,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/072b55ca7e6048f2a95f77f521b0c840 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/072b55ca7e6048f2a95f77f521b0c840 2024-11-25T05:55:07,181 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/072b55ca7e6048f2a95f77f521b0c840, entries=150, sequenceid=459, filesize=12.0 K 2024-11-25T05:55:07,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/6a4b3abbaab3403bbd7df5d64bca60e7 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/6a4b3abbaab3403bbd7df5d64bca60e7 2024-11-25T05:55:07,200 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/6a4b3abbaab3403bbd7df5d64bca60e7, entries=150, sequenceid=459, filesize=12.0 K 2024-11-25T05:55:07,203 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~429.28 KB/439584, currentSize=33.54 KB/34350 for cf8ce3f4f3ad5ac4b5297b0ca903cd34 in 567ms, sequenceid=459, compaction requested=true 2024-11-25T05:55:07,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:07,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:07,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-25T05:55:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-25T05:55:07,210 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-25T05:55:07,210 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2990 sec 2024-11-25T05:55:07,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.3080 sec 2024-11-25T05:55:07,290 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-25T05:55:07,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=A 2024-11-25T05:55:07,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:07,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=B 2024-11-25T05:55:07,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:07,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=C 2024-11-25T05:55:07,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:55:07,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/0e18801864004a17b71bbea64afe0ee2 is 50, key is test_row_0/A:col10/1732514107282/Put/seqid=0 2024-11-25T05:55:07,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741920_1096 (size=14741) 2024-11-25T05:55:07,363 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732514107337, max timestamp=1732514107360Num uniques 859; 2024-11-25T05:55:07,363 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-25T05:55:07,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732514107337, max timestamp=1732514107360Num uniques 859; 2024-11-25T05:55:07,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732514107337, max timestamp=1732514107360Num uniques 859; 2024-11-25T05:55:07,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-25T05:55:07,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-25T05:55:07,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:07,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514167439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:07,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:07,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514167442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:07,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:07,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:07,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:07,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:07,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514167443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:07,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:07,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:07,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:07,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:07,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514167562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:07,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:07,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:07,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:07,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:07,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514167563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:07,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:07,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:07,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:07,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:07,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514167563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:07,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:07,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:07,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:07,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/0e18801864004a17b71bbea64afe0ee2 2024-11-25T05:55:07,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/865741f1d5e44b0a8a26797cdefd571d is 50, key is test_row_0/B:col10/1732514107282/Put/seqid=0 2024-11-25T05:55:07,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:07,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514167782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:07,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:07,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:07,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:07,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514167782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:07,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:07,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:07,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:07,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:07,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:07,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514167783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:07,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:07,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:07,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:07,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741921_1097 (size=12301) 2024-11-25T05:55:07,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/865741f1d5e44b0a8a26797cdefd571d 2024-11-25T05:55:07,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/808f102497eb4e798adbc6dc2cda76ce is 50, key is test_row_0/C:col10/1732514107282/Put/seqid=0 2024-11-25T05:55:07,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741922_1098 (size=12301) 2024-11-25T05:55:07,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/808f102497eb4e798adbc6dc2cda76ce 2024-11-25T05:55:07,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/0e18801864004a17b71bbea64afe0ee2 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/0e18801864004a17b71bbea64afe0ee2 2024-11-25T05:55:07,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/0e18801864004a17b71bbea64afe0ee2, entries=200, sequenceid=469, filesize=14.4 K 2024-11-25T05:55:07,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/865741f1d5e44b0a8a26797cdefd571d as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/865741f1d5e44b0a8a26797cdefd571d 2024-11-25T05:55:07,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/865741f1d5e44b0a8a26797cdefd571d, entries=150, sequenceid=469, filesize=12.0 K 2024-11-25T05:55:07,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/808f102497eb4e798adbc6dc2cda76ce as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/808f102497eb4e798adbc6dc2cda76ce 2024-11-25T05:55:07,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/808f102497eb4e798adbc6dc2cda76ce, entries=150, sequenceid=469, filesize=12.0 K 2024-11-25T05:55:07,911 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for cf8ce3f4f3ad5ac4b5297b0ca903cd34 in 622ms, sequenceid=469, compaction requested=true 2024-11-25T05:55:07,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:07,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:55:07,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:07,911 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:07,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:55:07,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:07,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:55:07,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:07,912 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T05:55:07,916 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:07,916 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/A is initiating minor compaction (all files) 2024-11-25T05:55:07,916 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/A in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:07,916 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/80b0a231452446e2b5138a0368f038f7, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/c702903f5230430995613720e614581b, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/0e18801864004a17b71bbea64afe0ee2] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=39.3 K 2024-11-25T05:55:07,934 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80b0a231452446e2b5138a0368f038f7, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732514105357 2024-11-25T05:55:07,935 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting c702903f5230430995613720e614581b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1732514105759 2024-11-25T05:55:07,935 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T05:55:07,935 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/C is initiating minor compaction (all files) 2024-11-25T05:55:07,935 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/C in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:07,935 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/d4b15ed789c440ab9c9c46ee3cc6bfbe, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/81bde786ad2c435da00aee08ac9b5b52, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/6a4b3abbaab3403bbd7df5d64bca60e7, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/808f102497eb4e798adbc6dc2cda76ce] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=48.8 K 2024-11-25T05:55:07,936 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting d4b15ed789c440ab9c9c46ee3cc6bfbe, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732514104651 2024-11-25T05:55:07,936 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e18801864004a17b71bbea64afe0ee2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1732514106935 2024-11-25T05:55:07,937 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 81bde786ad2c435da00aee08ac9b5b52, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732514105357 2024-11-25T05:55:07,938 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a4b3abbaab3403bbd7df5d64bca60e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1732514105759 2024-11-25T05:55:07,938 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 808f102497eb4e798adbc6dc2cda76ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1732514106935 2024-11-25T05:55:07,960 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#A#compaction#86 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:07,961 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/fbaed385a5e14e3bad4fc4fdd9404ea9 is 50, key is test_row_0/A:col10/1732514107282/Put/seqid=0 2024-11-25T05:55:07,976 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#C#compaction#87 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:07,976 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/c9c5171dded440a7960130d43b85df98 is 50, key is test_row_0/C:col10/1732514107282/Put/seqid=0 2024-11-25T05:55:07,992 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-11-25T05:55:07,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:55:07,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732514107392, max timestamp=1732514107992Num uniques 1118; 2024-11-25T05:55:07,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732514107392, max timestamp=1732514107992Num uniques 1118; 2024-11-25T05:55:07,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-25T05:55:07,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-25T05:55:08,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732514107392, max timestamp=1732514107992Num uniques 1118; 2024-11-25T05:55:08,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-25T05:55:08,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=A 2024-11-25T05:55:08,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:55:08,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=B 2024-11-25T05:55:08,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:55:08,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=C 2024-11-25T05:55:08,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:55:08,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741923_1099 (size=13289) 2024-11-25T05:55:08,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741924_1100 (size=13255) 2024-11-25T05:55:08,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/e8dbab9aacac45a6af913230c98888ba is 50, key is test_row_0/A:col10/1732514107392/Put/seqid=0 2024-11-25T05:55:08,037 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/fbaed385a5e14e3bad4fc4fdd9404ea9 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/fbaed385a5e14e3bad4fc4fdd9404ea9 2024-11-25T05:55:08,047 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/A of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into fbaed385a5e14e3bad4fc4fdd9404ea9(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:08,047 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:08,048 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/A, priority=13, startTime=1732514107911; duration=0sec 2024-11-25T05:55:08,051 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:08,051 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:A 2024-11-25T05:55:08,051 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:08,053 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:08,053 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/B is initiating minor compaction (all files) 2024-11-25T05:55:08,053 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/B in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:08,053 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/7fe7cae59ee145fdac968a7b9d7767c5, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/072b55ca7e6048f2a95f77f521b0c840, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/865741f1d5e44b0a8a26797cdefd571d] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=36.9 K 2024-11-25T05:55:08,054 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fe7cae59ee145fdac968a7b9d7767c5, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732514105357 2024-11-25T05:55:08,055 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 072b55ca7e6048f2a95f77f521b0c840, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1732514105759 2024-11-25T05:55:08,056 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 865741f1d5e44b0a8a26797cdefd571d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1732514106935 2024-11-25T05:55:08,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741925_1101 (size=14741) 2024-11-25T05:55:08,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/e8dbab9aacac45a6af913230c98888ba 2024-11-25T05:55:08,083 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#B#compaction#89 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:08,084 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/582fb101d7ce4c04bc41bdbfef2ed6f6 is 50, key is test_row_0/B:col10/1732514107282/Put/seqid=0 2024-11-25T05:55:08,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/ffabb5839067498b8e2837fc1f3bff4d is 50, key is test_row_0/B:col10/1732514107392/Put/seqid=0 2024-11-25T05:55:08,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732514108054, max timestamp=1732514108107Num uniques 859; 2024-11-25T05:55:08,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T05:55:08,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732514108054, max timestamp=1732514108107Num uniques 859; 2024-11-25T05:55:08,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T05:55:08,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732514108054, max timestamp=1732514108107Num uniques 859; 2024-11-25T05:55:08,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T05:55:08,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741926_1102 (size=13289) 2024-11-25T05:55:08,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741927_1103 (size=12301) 2024-11-25T05:55:08,182 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/ffabb5839067498b8e2837fc1f3bff4d 2024-11-25T05:55:08,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/dc1166b8799f48a0bf16ab40a22fc971 is 50, key is test_row_0/C:col10/1732514107392/Put/seqid=0 2024-11-25T05:55:08,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732514108154, max timestamp=1732514108198Num uniques 1068; 2024-11-25T05:55:08,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-25T05:55:08,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732514108154, max timestamp=1732514108198Num uniques 1068; 2024-11-25T05:55:08,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-25T05:55:08,216 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732514108154, max timestamp=1732514108211Num uniques 1068; 2024-11-25T05:55:08,216 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-25T05:55:08,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741928_1104 (size=12301) 2024-11-25T05:55:08,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/dc1166b8799f48a0bf16ab40a22fc971 2024-11-25T05:55:08,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/e8dbab9aacac45a6af913230c98888ba as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/e8dbab9aacac45a6af913230c98888ba 2024-11-25T05:55:08,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/e8dbab9aacac45a6af913230c98888ba, entries=200, sequenceid=507, filesize=14.4 K 2024-11-25T05:55:08,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/ffabb5839067498b8e2837fc1f3bff4d as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/ffabb5839067498b8e2837fc1f3bff4d 2024-11-25T05:55:08,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732514108242, max timestamp=1732514108290Num uniques 1027; 2024-11-25T05:55:08,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-11-25T05:55:08,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/ffabb5839067498b8e2837fc1f3bff4d, entries=150, sequenceid=507, filesize=12.0 K 2024-11-25T05:55:08,298 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732514108269, max timestamp=1732514108290Num uniques 1027; 2024-11-25T05:55:08,299 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-25T05:55:08,299 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732514108269, max timestamp=1732514108295Num uniques 1027; 2024-11-25T05:55:08,299 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-25T05:55:08,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/dc1166b8799f48a0bf16ab40a22fc971 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/dc1166b8799f48a0bf16ab40a22fc971 2024-11-25T05:55:08,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/dc1166b8799f48a0bf16ab40a22fc971, entries=150, sequenceid=507, filesize=12.0 K 2024-11-25T05:55:08,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~80.58 KB/82512, currentSize=71.56 KB/73280 for cf8ce3f4f3ad5ac4b5297b0ca903cd34 in 324ms, sequenceid=507, compaction requested=false 2024-11-25T05:55:08,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:08,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:55:08,318 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34 3/3 column families, dataSize=78.27 KB heapSize=214.16 KB 2024-11-25T05:55:08,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=A 2024-11-25T05:55:08,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:55:08,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=B 2024-11-25T05:55:08,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:55:08,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=C 2024-11-25T05:55:08,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:55:08,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/61de2ff979bc491b84cc298079835f2f is 50, key is test_row_0/A:col10/1732514108316/Put/seqid=0 2024-11-25T05:55:08,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:08,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514168383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:08,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:08,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:08,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:08,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:08,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514168387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:08,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:08,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:08,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514168387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:08,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:08,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:08,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:08,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:08,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:08,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:08,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514168388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:08,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:08,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:08,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:08,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:08,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514168389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:08,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:08,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:08,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:08,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741929_1105 (size=19621) 2024-11-25T05:55:08,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/61de2ff979bc491b84cc298079835f2f 2024-11-25T05:55:08,434 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/c9c5171dded440a7960130d43b85df98 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/c9c5171dded440a7960130d43b85df98 2024-11-25T05:55:08,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/7b280806097a432d922d12e1ccf1df6a is 50, key is test_row_0/B:col10/1732514108316/Put/seqid=0 2024-11-25T05:55:08,446 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/C of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into c9c5171dded440a7960130d43b85df98(size=12.9 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:08,446 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:08,446 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/C, priority=12, startTime=1732514107912; duration=0sec 2024-11-25T05:55:08,447 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:08,447 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:C 2024-11-25T05:55:08,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741930_1106 (size=12301) 2024-11-25T05:55:08,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:08,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514168492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:08,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:08,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:08,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:08,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:08,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:08,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514168512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:08,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:08,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514168512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:08,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:08,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514168512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:08,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514168513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:08,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:08,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:08,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:08,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:08,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:08,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:08,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:08,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:08,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:08,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:08,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:08,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:08,553 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/582fb101d7ce4c04bc41bdbfef2ed6f6 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/582fb101d7ce4c04bc41bdbfef2ed6f6 2024-11-25T05:55:08,561 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/B of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into 582fb101d7ce4c04bc41bdbfef2ed6f6(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:08,562 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:08,562 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/B, priority=13, startTime=1732514107911; duration=0sec 2024-11-25T05:55:08,562 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:08,562 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:B 2024-11-25T05:55:08,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:08,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514168702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:08,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:08,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:08,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:08,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:08,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514168723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:08,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:08,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514168723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:08,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:08,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514168723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:08,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:08,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:08,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:08,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:08,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:08,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514168723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:08,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:08,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:08,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:08,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:08,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:08,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:08,885 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/7b280806097a432d922d12e1ccf1df6a 2024-11-25T05:55:08,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/7b5d6ec9962a412e93ae818cf8b0f278 is 50, key is test_row_0/C:col10/1732514108316/Put/seqid=0 2024-11-25T05:55:08,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741931_1107 (size=12301) 2024-11-25T05:55:08,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/7b5d6ec9962a412e93ae818cf8b0f278 2024-11-25T05:55:08,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/61de2ff979bc491b84cc298079835f2f as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/61de2ff979bc491b84cc298079835f2f 2024-11-25T05:55:08,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/61de2ff979bc491b84cc298079835f2f, entries=300, sequenceid=570, filesize=19.2 K 2024-11-25T05:55:08,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/7b280806097a432d922d12e1ccf1df6a as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/7b280806097a432d922d12e1ccf1df6a 2024-11-25T05:55:08,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/7b280806097a432d922d12e1ccf1df6a, entries=150, sequenceid=570, filesize=12.0 K 2024-11-25T05:55:08,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/7b5d6ec9962a412e93ae818cf8b0f278 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/7b5d6ec9962a412e93ae818cf8b0f278 2024-11-25T05:55:08,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/7b5d6ec9962a412e93ae818cf8b0f278, entries=150, sequenceid=570, filesize=12.0 K 2024-11-25T05:55:08,985 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~78.27 KB/80150, heapSize ~214.11 KB/219248, currentSize=100.63 KB/103050 for cf8ce3f4f3ad5ac4b5297b0ca903cd34 in 666ms, sequenceid=570, compaction requested=true 2024-11-25T05:55:08,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:08,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:55:08,985 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:08,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:08,985 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:08,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:55:08,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:08,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:55:08,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:08,987 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47651 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:08,987 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/A is initiating minor compaction (all files) 2024-11-25T05:55:08,987 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/A in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:08,987 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/fbaed385a5e14e3bad4fc4fdd9404ea9, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/e8dbab9aacac45a6af913230c98888ba, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/61de2ff979bc491b84cc298079835f2f] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=46.5 K 2024-11-25T05:55:08,988 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:08,988 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/B is initiating minor compaction (all files) 2024-11-25T05:55:08,988 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/B in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:08,988 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/582fb101d7ce4c04bc41bdbfef2ed6f6, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/ffabb5839067498b8e2837fc1f3bff4d, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/7b280806097a432d922d12e1ccf1df6a] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=37.0 K 2024-11-25T05:55:08,994 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbaed385a5e14e3bad4fc4fdd9404ea9, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1732514106935 2024-11-25T05:55:08,996 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 582fb101d7ce4c04bc41bdbfef2ed6f6, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1732514106935 2024-11-25T05:55:08,996 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8dbab9aacac45a6af913230c98888ba, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1732514107392 2024-11-25T05:55:09,000 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting ffabb5839067498b8e2837fc1f3bff4d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1732514107392 2024-11-25T05:55:09,000 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61de2ff979bc491b84cc298079835f2f, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1732514108269 2024-11-25T05:55:09,000 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b280806097a432d922d12e1ccf1df6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1732514108269 2024-11-25T05:55:09,024 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#B#compaction#96 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:09,024 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#A#compaction#95 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:09,025 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/79a51fa6fc214414ac6c1ce9de9bac0c is 50, key is test_row_0/A:col10/1732514108316/Put/seqid=0 2024-11-25T05:55:09,025 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/7ad83bd490bc471c84f518d3df630d08 is 50, key is test_row_0/B:col10/1732514108316/Put/seqid=0 2024-11-25T05:55:09,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:55:09,028 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-25T05:55:09,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=A 2024-11-25T05:55:09,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:09,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=B 2024-11-25T05:55:09,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:09,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=C 2024-11-25T05:55:09,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:09,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/4aca44e51c7c4f67aec1d74c77327f64 is 50, key is test_row_0/A:col10/1732514108376/Put/seqid=0 2024-11-25T05:55:09,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:09,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:09,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514169048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:09,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514169050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:09,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741932_1108 (size=13391) 2024-11-25T05:55:09,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:09,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514169052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:09,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:09,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:09,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:09,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:09,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514169054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:09,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:09,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514169055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:09,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:09,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:09,065 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/79a51fa6fc214414ac6c1ce9de9bac0c as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/79a51fa6fc214414ac6c1ce9de9bac0c 2024-11-25T05:55:09,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741933_1109 (size=13391) 2024-11-25T05:55:09,081 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/A of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into 79a51fa6fc214414ac6c1ce9de9bac0c(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:09,081 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:09,081 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/A, priority=13, startTime=1732514108985; duration=0sec 2024-11-25T05:55:09,081 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:09,081 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:A 2024-11-25T05:55:09,081 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:09,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T05:55:09,082 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:55:09,083 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:55:09,085 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:09,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-25T05:55:09,085 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/C is initiating minor compaction (all files) 2024-11-25T05:55:09,085 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/C in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:09,085 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/c9c5171dded440a7960130d43b85df98, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/dc1166b8799f48a0bf16ab40a22fc971, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/7b5d6ec9962a412e93ae818cf8b0f278] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=37.0 K 2024-11-25T05:55:09,087 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:55:09,087 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9c5171dded440a7960130d43b85df98, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1732514106935 2024-11-25T05:55:09,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T05:55:09,088 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:55:09,088 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:55:09,088 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc1166b8799f48a0bf16ab40a22fc971, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1732514107392 2024-11-25T05:55:09,089 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b5d6ec9962a412e93ae818cf8b0f278, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1732514108269 2024-11-25T05:55:09,092 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/7ad83bd490bc471c84f518d3df630d08 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/7ad83bd490bc471c84f518d3df630d08 2024-11-25T05:55:09,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741934_1110 (size=12301) 2024-11-25T05:55:09,102 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=591 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/4aca44e51c7c4f67aec1d74c77327f64 2024-11-25T05:55:09,115 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/B of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into 7ad83bd490bc471c84f518d3df630d08(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:09,116 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:09,116 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/B, priority=13, startTime=1732514108985; duration=0sec 2024-11-25T05:55:09,116 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:09,116 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:B 2024-11-25T05:55:09,119 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#C#compaction#98 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:09,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/f247339faffe4d65920953c7bb1f2d84 is 50, key is test_row_0/B:col10/1732514108376/Put/seqid=0 2024-11-25T05:55:09,123 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/5aba07e9c90a428b8f25d7013d2abf6e is 50, key is test_row_0/C:col10/1732514108316/Put/seqid=0 2024-11-25T05:55:09,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741936_1112 (size=13357) 2024-11-25T05:55:09,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741935_1111 (size=12301) 2024-11-25T05:55:09,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=591 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/f247339faffe4d65920953c7bb1f2d84 2024-11-25T05:55:09,158 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/5aba07e9c90a428b8f25d7013d2abf6e as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/5aba07e9c90a428b8f25d7013d2abf6e 2024-11-25T05:55:09,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:09,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514169162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:09,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:09,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:09,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514169162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:09,170 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/C of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into 5aba07e9c90a428b8f25d7013d2abf6e(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:09,170 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:09,170 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/C, priority=13, startTime=1732514108986; duration=0sec 2024-11-25T05:55:09,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:09,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,170 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:09,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514169163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:09,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:09,170 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:C 2024-11-25T05:55:09,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:09,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/d225455dd4dc4e52a0e0f50403c1ceec is 50, key is test_row_0/C:col10/1732514108376/Put/seqid=0 2024-11-25T05:55:09,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:09,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514169172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:09,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514169172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:09,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T05:55:09,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741937_1113 (size=12301) 2024-11-25T05:55:09,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T05:55:09,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:09,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:09,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:09,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:09,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:09,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514169373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:09,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:09,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514169382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:09,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:09,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:09,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514169382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514169382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:09,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:09,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514169382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:09,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:09,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:09,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T05:55:09,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:09,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:09,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:09,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:09,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:09,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:09,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T05:55:09,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T05:55:09,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:09,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:09,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:09,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:09,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:09,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:09,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=591 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/d225455dd4dc4e52a0e0f50403c1ceec 2024-11-25T05:55:09,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/4aca44e51c7c4f67aec1d74c77327f64 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/4aca44e51c7c4f67aec1d74c77327f64 2024-11-25T05:55:09,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/4aca44e51c7c4f67aec1d74c77327f64, entries=150, sequenceid=591, filesize=12.0 K 2024-11-25T05:55:09,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/f247339faffe4d65920953c7bb1f2d84 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/f247339faffe4d65920953c7bb1f2d84 2024-11-25T05:55:09,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/f247339faffe4d65920953c7bb1f2d84, entries=150, sequenceid=591, filesize=12.0 K 2024-11-25T05:55:09,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/d225455dd4dc4e52a0e0f50403c1ceec as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/d225455dd4dc4e52a0e0f50403c1ceec 2024-11-25T05:55:09,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/d225455dd4dc4e52a0e0f50403c1ceec, entries=150, sequenceid=591, filesize=12.0 K 2024-11-25T05:55:09,675 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for cf8ce3f4f3ad5ac4b5297b0ca903cd34 in 647ms, sequenceid=591, compaction requested=false 2024-11-25T05:55:09,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:09,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:55:09,685 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-25T05:55:09,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=A 2024-11-25T05:55:09,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:09,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=B 2024-11-25T05:55:09,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:09,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=C 2024-11-25T05:55:09,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:09,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/ca1255e514fc4cd59cc216374d25cfea is 50, key is test_row_0/A:col10/1732514109683/Put/seqid=0 2024-11-25T05:55:09,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741938_1114 (size=14741) 2024-11-25T05:55:09,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T05:55:09,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:09,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:09,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:09,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:09,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:09,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:09,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T05:55:09,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:09,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514169709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:09,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:09,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514169711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:09,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:09,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:09,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:09,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514169721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:09,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:09,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:09,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514169721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:09,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:09,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:09,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514169724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:09,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:09,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:09,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514169833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:09,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:09,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:09,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514169833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:09,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:09,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:09,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514169842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:09,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:09,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514169843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:09,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:09,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514169844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:09,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:09,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:09,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:09,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T05:55:09,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:09,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:09,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:09,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:09,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:09,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:10,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T05:55:10,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:10,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:10,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:10,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:10,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:10,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:10,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:10,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514170042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:10,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:10,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:10,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:10,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:10,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514170052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:10,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:10,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:10,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:10,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514170052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:10,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:10,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:10,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514170053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:10,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:10,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:10,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:10,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:10,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514170053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:10,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:10,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:10,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:10,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:10,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:10,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:10,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=609 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/ca1255e514fc4cd59cc216374d25cfea 2024-11-25T05:55:10,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/60d039bfa017403f9d61879227ea194a is 50, key is test_row_0/B:col10/1732514109683/Put/seqid=0 2024-11-25T05:55:10,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741939_1115 (size=12301) 2024-11-25T05:55:10,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=609 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/60d039bfa017403f9d61879227ea194a 2024-11-25T05:55:10,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/24f36679b0154aff9a2f07f8b031a706 is 50, key is test_row_0/C:col10/1732514109683/Put/seqid=0 2024-11-25T05:55:10,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T05:55:10,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:10,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:10,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:10,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:10,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:10,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:10,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741940_1116 (size=12301) 2024-11-25T05:55:10,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T05:55:10,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T05:55:10,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:10,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:10,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:10,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:10,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:10,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:10,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:10,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514170352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:10,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:10,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:10,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:10,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:10,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514170363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:10,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:10,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514170363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:10,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:10,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514170363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:10,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:10,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:10,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:10,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:10,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:10,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:10,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:10,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514170363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:10,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:10,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:10,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:10,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:10,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:10,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:10,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T05:55:10,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:10,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:10,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:10,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:10,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:10,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:10,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=609 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/24f36679b0154aff9a2f07f8b031a706 2024-11-25T05:55:10,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/ca1255e514fc4cd59cc216374d25cfea as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/ca1255e514fc4cd59cc216374d25cfea 2024-11-25T05:55:10,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/ca1255e514fc4cd59cc216374d25cfea, entries=200, sequenceid=609, filesize=14.4 K 2024-11-25T05:55:10,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/60d039bfa017403f9d61879227ea194a as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/60d039bfa017403f9d61879227ea194a 2024-11-25T05:55:10,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/60d039bfa017403f9d61879227ea194a, entries=150, sequenceid=609, filesize=12.0 K 2024-11-25T05:55:10,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/24f36679b0154aff9a2f07f8b031a706 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/24f36679b0154aff9a2f07f8b031a706 2024-11-25T05:55:10,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T05:55:10,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:10,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:10,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:10,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:10,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:10,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:10,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/24f36679b0154aff9a2f07f8b031a706, entries=150, sequenceid=609, filesize=12.0 K 2024-11-25T05:55:10,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for cf8ce3f4f3ad5ac4b5297b0ca903cd34 in 952ms, sequenceid=609, compaction requested=true 2024-11-25T05:55:10,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:10,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:55:10,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:10,638 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:10,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:55:10,638 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:10,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:10,640 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:10,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:55:10,640 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/A is initiating minor compaction (all files) 2024-11-25T05:55:10,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:10,640 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/A in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:10,641 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/79a51fa6fc214414ac6c1ce9de9bac0c, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/4aca44e51c7c4f67aec1d74c77327f64, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/ca1255e514fc4cd59cc216374d25cfea] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=39.5 K 2024-11-25T05:55:10,641 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:10,641 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/B is initiating minor compaction (all files) 2024-11-25T05:55:10,641 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/B in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:10,641 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79a51fa6fc214414ac6c1ce9de9bac0c, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1732514108269 2024-11-25T05:55:10,641 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/7ad83bd490bc471c84f518d3df630d08, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/f247339faffe4d65920953c7bb1f2d84, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/60d039bfa017403f9d61879227ea194a] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=37.1 K 2024-11-25T05:55:10,642 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4aca44e51c7c4f67aec1d74c77327f64, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1732514108376 2024-11-25T05:55:10,642 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ad83bd490bc471c84f518d3df630d08, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1732514108269 2024-11-25T05:55:10,642 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca1255e514fc4cd59cc216374d25cfea, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=609, earliestPutTs=1732514109036 2024-11-25T05:55:10,642 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting f247339faffe4d65920953c7bb1f2d84, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1732514108376 2024-11-25T05:55:10,643 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 60d039bfa017403f9d61879227ea194a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=609, earliestPutTs=1732514109048 2024-11-25T05:55:10,662 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#A#compaction#104 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:10,663 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/1aa915c43d2144c0b0b5f069dd817eb1 is 50, key is test_row_0/A:col10/1732514109683/Put/seqid=0 2024-11-25T05:55:10,667 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#B#compaction#105 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:10,668 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/3068d9223dde48b3929434f688953ca4 is 50, key is test_row_0/B:col10/1732514109683/Put/seqid=0 2024-11-25T05:55:10,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741941_1117 (size=13493) 2024-11-25T05:55:10,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741942_1118 (size=13493) 2024-11-25T05:55:10,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T05:55:10,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:10,787 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-25T05:55:10,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=A 2024-11-25T05:55:10,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:10,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=B 2024-11-25T05:55:10,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:10,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=C 2024-11-25T05:55:10,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:10,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/d993fee514df43ef951abd5e3bacfd20 is 50, key is test_row_0/A:col10/1732514109721/Put/seqid=0 2024-11-25T05:55:10,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741943_1119 (size=12301) 2024-11-25T05:55:10,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:55:10,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:10,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:10,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:10,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514170887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:10,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514170889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:10,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:10,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:10,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:10,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:10,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:10,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514170891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:10,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:10,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:10,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:10,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:10,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:10,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:10,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514170899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:10,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:10,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:10,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:10,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:10,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514170901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:10,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:10,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:10,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:11,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:11,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514171002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:11,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:11,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:11,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:11,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514171002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:11,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:11,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:11,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:11,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:11,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:11,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514171002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:11,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:11,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:11,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514171002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:11,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:11,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:11,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:11,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:11,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:11,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:11,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514171011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:11,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:11,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:11,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:11,094 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/1aa915c43d2144c0b0b5f069dd817eb1 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/1aa915c43d2144c0b0b5f069dd817eb1 2024-11-25T05:55:11,107 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/A of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into 1aa915c43d2144c0b0b5f069dd817eb1(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:11,107 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:11,107 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/A, priority=13, startTime=1732514110638; duration=0sec 2024-11-25T05:55:11,107 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:11,107 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:A 2024-11-25T05:55:11,107 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:11,110 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/3068d9223dde48b3929434f688953ca4 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/3068d9223dde48b3929434f688953ca4 2024-11-25T05:55:11,133 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:11,134 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/C is initiating minor compaction (all files) 2024-11-25T05:55:11,134 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/C in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:11,135 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/5aba07e9c90a428b8f25d7013d2abf6e, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/d225455dd4dc4e52a0e0f50403c1ceec, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/24f36679b0154aff9a2f07f8b031a706] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=37.1 K 2024-11-25T05:55:11,140 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/B of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into 3068d9223dde48b3929434f688953ca4(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:11,140 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:11,140 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/B, priority=13, startTime=1732514110638; duration=0sec 2024-11-25T05:55:11,140 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:11,140 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:B 2024-11-25T05:55:11,146 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5aba07e9c90a428b8f25d7013d2abf6e, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1732514108269 2024-11-25T05:55:11,153 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting d225455dd4dc4e52a0e0f50403c1ceec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1732514108376 2024-11-25T05:55:11,154 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24f36679b0154aff9a2f07f8b031a706, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=609, earliestPutTs=1732514109048 2024-11-25T05:55:11,196 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#C#compaction#107 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:11,197 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/944ee2589adf45b3a8b5662fd9975ae5 is 50, key is test_row_0/C:col10/1732514109683/Put/seqid=0 2024-11-25T05:55:11,206 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=626 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/d993fee514df43ef951abd5e3bacfd20 2024-11-25T05:55:11,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514171212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:11,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:11,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:11,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:11,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:11,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514171213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:11,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:11,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:11,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:11,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:11,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514171213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:11,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:11,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:11,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:11,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:11,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514171213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:11,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:11,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:11,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:11,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:11,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514171222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:11,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:11,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:11,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:11,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T05:55:11,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/1932ae9dbdc7460a984e8fc0a3f9a882 is 50, key is test_row_0/B:col10/1732514109721/Put/seqid=0 2024-11-25T05:55:11,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741944_1120 (size=13459) 2024-11-25T05:55:11,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741945_1121 (size=12301) 2024-11-25T05:55:11,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:11,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514171522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:11,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:11,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514171522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:11,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:11,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:11,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:11,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:11,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:11,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:11,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514171522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:11,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:11,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:11,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:11,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:11,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514171522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:11,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:11,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:11,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:11,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:11,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:11,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514171532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:11,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:11,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:11,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:11,660 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/944ee2589adf45b3a8b5662fd9975ae5 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/944ee2589adf45b3a8b5662fd9975ae5 2024-11-25T05:55:11,670 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/C of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into 944ee2589adf45b3a8b5662fd9975ae5(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:11,670 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:11,670 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/C, priority=13, startTime=1732514110638; duration=0sec 2024-11-25T05:55:11,670 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:11,670 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:C 2024-11-25T05:55:11,713 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=626 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/1932ae9dbdc7460a984e8fc0a3f9a882 2024-11-25T05:55:11,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/6e204afd32544324b1a5cc157d33855c is 50, key is test_row_0/C:col10/1732514109721/Put/seqid=0 2024-11-25T05:55:11,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741946_1122 (size=12301) 2024-11-25T05:55:11,734 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=626 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/6e204afd32544324b1a5cc157d33855c 2024-11-25T05:55:11,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/d993fee514df43ef951abd5e3bacfd20 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/d993fee514df43ef951abd5e3bacfd20 2024-11-25T05:55:11,750 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/d993fee514df43ef951abd5e3bacfd20, entries=150, sequenceid=626, filesize=12.0 K 2024-11-25T05:55:11,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/1932ae9dbdc7460a984e8fc0a3f9a882 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/1932ae9dbdc7460a984e8fc0a3f9a882 2024-11-25T05:55:11,757 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/1932ae9dbdc7460a984e8fc0a3f9a882, entries=150, sequenceid=626, filesize=12.0 K 2024-11-25T05:55:11,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/6e204afd32544324b1a5cc157d33855c as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/6e204afd32544324b1a5cc157d33855c 2024-11-25T05:55:11,765 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/6e204afd32544324b1a5cc157d33855c, entries=150, sequenceid=626, filesize=12.0 K 2024-11-25T05:55:11,767 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for cf8ce3f4f3ad5ac4b5297b0ca903cd34 in 980ms, sequenceid=626, compaction requested=false 2024-11-25T05:55:11,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:11,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:11,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-25T05:55:11,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-25T05:55:11,770 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-25T05:55:11,771 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6810 sec 2024-11-25T05:55:11,776 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.6910 sec 2024-11-25T05:55:12,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-25T05:55:12,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:55:12,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=A 2024-11-25T05:55:12,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:12,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=B 2024-11-25T05:55:12,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:12,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=C 2024-11-25T05:55:12,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:12,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/85a4122fa7df4236bb70a0553f678291 is 50, key is test_row_0/A:col10/1732514110890/Put/seqid=0 2024-11-25T05:55:12,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:12,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514172059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:12,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:12,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:12,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514172062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:12,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:12,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514172062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:12,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:12,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:12,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:12,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514172065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:12,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:12,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514172065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:12,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:12,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:12,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741947_1123 (size=12301) 2024-11-25T05:55:12,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=646 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/85a4122fa7df4236bb70a0553f678291 2024-11-25T05:55:12,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/b40b12b0f65f411eadfdd143111dafaa is 50, key is test_row_0/B:col10/1732514110890/Put/seqid=0 2024-11-25T05:55:12,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741948_1124 (size=12301) 2024-11-25T05:55:12,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=646 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/b40b12b0f65f411eadfdd143111dafaa 2024-11-25T05:55:12,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/97b5c25ea13f43e2a78196e3d7b6dc50 is 50, key is test_row_0/C:col10/1732514110890/Put/seqid=0 2024-11-25T05:55:12,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:12,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514172172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:12,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:12,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:12,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514172182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:12,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:12,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:12,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514172182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:12,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:12,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514172183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:12,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:12,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:12,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:12,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514172183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:12,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:12,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741949_1125 (size=12301) 2024-11-25T05:55:12,199 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=646 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/97b5c25ea13f43e2a78196e3d7b6dc50 2024-11-25T05:55:12,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/85a4122fa7df4236bb70a0553f678291 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/85a4122fa7df4236bb70a0553f678291 2024-11-25T05:55:12,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/85a4122fa7df4236bb70a0553f678291, entries=150, sequenceid=646, filesize=12.0 K 2024-11-25T05:55:12,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/b40b12b0f65f411eadfdd143111dafaa as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/b40b12b0f65f411eadfdd143111dafaa 2024-11-25T05:55:12,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/b40b12b0f65f411eadfdd143111dafaa, entries=150, sequenceid=646, filesize=12.0 K 2024-11-25T05:55:12,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/97b5c25ea13f43e2a78196e3d7b6dc50 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/97b5c25ea13f43e2a78196e3d7b6dc50 2024-11-25T05:55:12,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/97b5c25ea13f43e2a78196e3d7b6dc50, entries=150, sequenceid=646, filesize=12.0 K 2024-11-25T05:55:12,253 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for cf8ce3f4f3ad5ac4b5297b0ca903cd34 in 216ms, sequenceid=646, compaction requested=true 2024-11-25T05:55:12,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:12,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:55:12,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:12,253 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:12,254 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:12,255 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:12,255 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/A is initiating minor compaction (all files) 2024-11-25T05:55:12,255 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/A in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:12,255 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/1aa915c43d2144c0b0b5f069dd817eb1, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/d993fee514df43ef951abd5e3bacfd20, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/85a4122fa7df4236bb70a0553f678291] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=37.2 K 2024-11-25T05:55:12,256 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1aa915c43d2144c0b0b5f069dd817eb1, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=609, earliestPutTs=1732514109048 2024-11-25T05:55:12,257 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:12,257 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/B is initiating minor compaction (all files) 2024-11-25T05:55:12,257 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting d993fee514df43ef951abd5e3bacfd20, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=626, earliestPutTs=1732514109707 2024-11-25T05:55:12,257 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/B in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:12,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:55:12,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:12,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:55:12,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:12,258 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/3068d9223dde48b3929434f688953ca4, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/1932ae9dbdc7460a984e8fc0a3f9a882, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/b40b12b0f65f411eadfdd143111dafaa] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=37.2 K 2024-11-25T05:55:12,260 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 3068d9223dde48b3929434f688953ca4, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=609, earliestPutTs=1732514109048 2024-11-25T05:55:12,260 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85a4122fa7df4236bb70a0553f678291, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=646, earliestPutTs=1732514110887 2024-11-25T05:55:12,260 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 1932ae9dbdc7460a984e8fc0a3f9a882, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=626, earliestPutTs=1732514109707 2024-11-25T05:55:12,261 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting b40b12b0f65f411eadfdd143111dafaa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=646, earliestPutTs=1732514110887 2024-11-25T05:55:12,288 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#A#compaction#113 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:12,288 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/aa42e43dd4064cea98bde5417faf425f is 50, key is test_row_0/A:col10/1732514110890/Put/seqid=0 2024-11-25T05:55:12,296 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#B#compaction#114 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:12,297 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/2809d3d167aa45889dfe80f3d54c9a32 is 50, key is test_row_0/B:col10/1732514110890/Put/seqid=0 2024-11-25T05:55:12,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741950_1126 (size=13595) 2024-11-25T05:55:12,329 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/aa42e43dd4064cea98bde5417faf425f as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/aa42e43dd4064cea98bde5417faf425f 2024-11-25T05:55:12,338 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/A of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into aa42e43dd4064cea98bde5417faf425f(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:12,339 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:12,339 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/A, priority=13, startTime=1732514112253; duration=0sec 2024-11-25T05:55:12,339 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:12,339 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:A 2024-11-25T05:55:12,339 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:12,341 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:12,341 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/C is initiating minor compaction (all files) 2024-11-25T05:55:12,341 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/C in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:12,341 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/944ee2589adf45b3a8b5662fd9975ae5, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/6e204afd32544324b1a5cc157d33855c, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/97b5c25ea13f43e2a78196e3d7b6dc50] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=37.2 K 2024-11-25T05:55:12,343 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 944ee2589adf45b3a8b5662fd9975ae5, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=609, earliestPutTs=1732514109048 2024-11-25T05:55:12,344 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e204afd32544324b1a5cc157d33855c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=626, earliestPutTs=1732514109707 2024-11-25T05:55:12,345 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97b5c25ea13f43e2a78196e3d7b6dc50, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=646, earliestPutTs=1732514110887 2024-11-25T05:55:12,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741951_1127 (size=13595) 2024-11-25T05:55:12,380 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#C#compaction#115 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:12,380 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/2809d3d167aa45889dfe80f3d54c9a32 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/2809d3d167aa45889dfe80f3d54c9a32 2024-11-25T05:55:12,380 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/6e79f5038205497c9c2177cbc0556854 is 50, key is test_row_0/C:col10/1732514110890/Put/seqid=0 2024-11-25T05:55:12,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-25T05:55:12,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=A 2024-11-25T05:55:12,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:12,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=B 2024-11-25T05:55:12,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:12,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=C 2024-11-25T05:55:12,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:12,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:55:12,393 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/B of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into 2809d3d167aa45889dfe80f3d54c9a32(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:12,393 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:12,393 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/B, priority=13, startTime=1732514112253; duration=0sec 2024-11-25T05:55:12,393 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:12,393 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:B 2024-11-25T05:55:12,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/41725bc2863f48a8bcadc164237a0543 is 50, key is test_row_0/A:col10/1732514112382/Put/seqid=0 2024-11-25T05:55:12,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741952_1128 (size=13561) 2024-11-25T05:55:12,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:12,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514172411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:12,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:12,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:12,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514172411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:12,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514172414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:12,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:12,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:12,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514172414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:12,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:12,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:12,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:12,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:12,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514172420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:12,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:12,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741953_1129 (size=14741) 2024-11-25T05:55:12,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=664 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/41725bc2863f48a8bcadc164237a0543 2024-11-25T05:55:12,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/c5ef8fc34fab4e519d627b223e028e6e is 50, key is test_row_0/B:col10/1732514112382/Put/seqid=0 2024-11-25T05:55:12,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741954_1130 (size=12301) 2024-11-25T05:55:12,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=664 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/c5ef8fc34fab4e519d627b223e028e6e 2024-11-25T05:55:12,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/81f029697c0d475bb395f6b4d32676b6 is 50, key is test_row_0/C:col10/1732514112382/Put/seqid=0 2024-11-25T05:55:12,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514172532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:12,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:12,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514172532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:12,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:12,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514172532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:12,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:12,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:12,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:12,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514172532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:12,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:12,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:12,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514172532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:12,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:12,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741955_1131 (size=12301) 2024-11-25T05:55:12,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:12,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514172743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:12,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:12,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:12,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514172743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514172743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:12,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514172743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:12,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:12,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:12,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514172752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:12,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:12,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:12,824 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/6e79f5038205497c9c2177cbc0556854 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/6e79f5038205497c9c2177cbc0556854 2024-11-25T05:55:12,833 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/C of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into 6e79f5038205497c9c2177cbc0556854(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:12,833 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:12,836 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/C, priority=13, startTime=1732514112257; duration=0sec 2024-11-25T05:55:12,836 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:12,836 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:C 2024-11-25T05:55:12,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=664 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/81f029697c0d475bb395f6b4d32676b6 2024-11-25T05:55:12,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/41725bc2863f48a8bcadc164237a0543 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/41725bc2863f48a8bcadc164237a0543 2024-11-25T05:55:12,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/41725bc2863f48a8bcadc164237a0543, entries=200, sequenceid=664, filesize=14.4 K 2024-11-25T05:55:12,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/c5ef8fc34fab4e519d627b223e028e6e as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/c5ef8fc34fab4e519d627b223e028e6e 2024-11-25T05:55:12,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/c5ef8fc34fab4e519d627b223e028e6e, entries=150, sequenceid=664, filesize=12.0 K 2024-11-25T05:55:12,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/81f029697c0d475bb395f6b4d32676b6 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/81f029697c0d475bb395f6b4d32676b6 2024-11-25T05:55:12,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/81f029697c0d475bb395f6b4d32676b6, entries=150, sequenceid=664, filesize=12.0 K 2024-11-25T05:55:12,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for cf8ce3f4f3ad5ac4b5297b0ca903cd34 in 613ms, sequenceid=664, compaction requested=false 2024-11-25T05:55:12,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:13,055 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-25T05:55:13,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=A 2024-11-25T05:55:13,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:13,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=B 2024-11-25T05:55:13,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:13,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=C 2024-11-25T05:55:13,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:13,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:55:13,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/659b7bdb5c944fe19786af6dae75c668 is 50, key is test_row_0/A:col10/1732514113053/Put/seqid=0 2024-11-25T05:55:13,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:13,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514173083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:13,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:13,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:13,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514173084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:13,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:13,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514173093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:13,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:13,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:13,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:13,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514173094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:13,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:13,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:13,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514173097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:13,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:13,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741956_1132 (size=14741) 2024-11-25T05:55:13,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=682 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/659b7bdb5c944fe19786af6dae75c668 2024-11-25T05:55:13,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/cf3246d2542547aba6e302564ccc68d9 is 50, key is test_row_0/B:col10/1732514113053/Put/seqid=0 2024-11-25T05:55:13,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741957_1133 (size=12301) 2024-11-25T05:55:13,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:13,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514173202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:13,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:13,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:13,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514173212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:13,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:13,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:13,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514173213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:13,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:13,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:13,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514173213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:13,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:13,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514173213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:13,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:13,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:13,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T05:55:13,252 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:55:13,253 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:55:13,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-25T05:55:13,256 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:55:13,257 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:55:13,257 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:55:13,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-25T05:55:13,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-25T05:55:13,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-25T05:55:13,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:13,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:13,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:13,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:13,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:13,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:13,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:13,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514173412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:13,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:13,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514173422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:13,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:13,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514173422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:13,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:13,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:13,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:13,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514173436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:13,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:13,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:13,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514173436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:13,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-25T05:55:13,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:13,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:13,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:13,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:13,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:13,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:13,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-25T05:55:13,603 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=682 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/cf3246d2542547aba6e302564ccc68d9 2024-11-25T05:55:13,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/89396391b4c941c799ad49fcd6919093 is 50, key is test_row_0/C:col10/1732514113053/Put/seqid=0 2024-11-25T05:55:13,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741958_1134 (size=12301) 2024-11-25T05:55:13,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=682 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/89396391b4c941c799ad49fcd6919093 2024-11-25T05:55:13,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/659b7bdb5c944fe19786af6dae75c668 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/659b7bdb5c944fe19786af6dae75c668 2024-11-25T05:55:13,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/659b7bdb5c944fe19786af6dae75c668, entries=200, sequenceid=682, filesize=14.4 K 2024-11-25T05:55:13,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/cf3246d2542547aba6e302564ccc68d9 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/cf3246d2542547aba6e302564ccc68d9 2024-11-25T05:55:13,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/cf3246d2542547aba6e302564ccc68d9, entries=150, sequenceid=682, filesize=12.0 K 2024-11-25T05:55:13,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/89396391b4c941c799ad49fcd6919093 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/89396391b4c941c799ad49fcd6919093 2024-11-25T05:55:13,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/89396391b4c941c799ad49fcd6919093, entries=150, sequenceid=682, filesize=12.0 K 2024-11-25T05:55:13,696 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for cf8ce3f4f3ad5ac4b5297b0ca903cd34 in 641ms, sequenceid=682, compaction requested=true 2024-11-25T05:55:13,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:13,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:55:13,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:13,697 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:13,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:55:13,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:13,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:55:13,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T05:55:13,697 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:13,698 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43077 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:13,698 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/A is initiating minor compaction (all files) 2024-11-25T05:55:13,699 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/A in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:13,699 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/aa42e43dd4064cea98bde5417faf425f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/41725bc2863f48a8bcadc164237a0543, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/659b7bdb5c944fe19786af6dae75c668] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=42.1 K 2024-11-25T05:55:13,699 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:13,699 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/B is initiating minor compaction (all files) 2024-11-25T05:55:13,699 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/B in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:13,699 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/2809d3d167aa45889dfe80f3d54c9a32, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/c5ef8fc34fab4e519d627b223e028e6e, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/cf3246d2542547aba6e302564ccc68d9] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=37.3 K 2024-11-25T05:55:13,699 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa42e43dd4064cea98bde5417faf425f, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=646, earliestPutTs=1732514110887 2024-11-25T05:55:13,700 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 2809d3d167aa45889dfe80f3d54c9a32, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=646, earliestPutTs=1732514110887 2024-11-25T05:55:13,700 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41725bc2863f48a8bcadc164237a0543, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=664, earliestPutTs=1732514112038 2024-11-25T05:55:13,700 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 659b7bdb5c944fe19786af6dae75c668, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=682, earliestPutTs=1732514112408 2024-11-25T05:55:13,700 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting c5ef8fc34fab4e519d627b223e028e6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=664, earliestPutTs=1732514112038 2024-11-25T05:55:13,701 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting cf3246d2542547aba6e302564ccc68d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=682, earliestPutTs=1732514112408 2024-11-25T05:55:13,717 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#B#compaction#122 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:13,718 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/b829be5d7ec1493db5f7ad49252af3dc is 50, key is test_row_0/B:col10/1732514113053/Put/seqid=0 2024-11-25T05:55:13,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-25T05:55:13,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:13,719 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-25T05:55:13,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=A 2024-11-25T05:55:13,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:13,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=B 2024-11-25T05:55:13,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:13,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=C 2024-11-25T05:55:13,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:13,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:55:13,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:13,734 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#A#compaction#123 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:13,735 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/6a9b34bdc2494a49a7a78085487baed0 is 50, key is test_row_0/A:col10/1732514113053/Put/seqid=0 2024-11-25T05:55:13,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/696f388190f7477c8cdc048c30324f75 is 50, key is test_row_0/A:col10/1732514113083/Put/seqid=0 2024-11-25T05:55:13,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741959_1135 (size=13697) 2024-11-25T05:55:13,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:13,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514173761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:13,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:13,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514173763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:13,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:13,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:13,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:13,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514173774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:13,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:13,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514173775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:13,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:13,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:13,787 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/b829be5d7ec1493db5f7ad49252af3dc as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/b829be5d7ec1493db5f7ad49252af3dc 2024-11-25T05:55:13,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741960_1136 (size=13697) 2024-11-25T05:55:13,802 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/B of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into b829be5d7ec1493db5f7ad49252af3dc(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:13,802 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:13,802 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/B, priority=13, startTime=1732514113697; duration=0sec 2024-11-25T05:55:13,803 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:13,803 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:B 2024-11-25T05:55:13,804 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:13,808 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38163 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:13,808 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/C is initiating minor compaction (all files) 2024-11-25T05:55:13,808 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/C in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:13,808 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/6e79f5038205497c9c2177cbc0556854, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/81f029697c0d475bb395f6b4d32676b6, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/89396391b4c941c799ad49fcd6919093] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=37.3 K 2024-11-25T05:55:13,809 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e79f5038205497c9c2177cbc0556854, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=646, earliestPutTs=1732514110887 2024-11-25T05:55:13,810 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 81f029697c0d475bb395f6b4d32676b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=664, earliestPutTs=1732514112038 2024-11-25T05:55:13,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:13,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514173786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:13,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:13,812 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 89396391b4c941c799ad49fcd6919093, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=682, earliestPutTs=1732514112408 2024-11-25T05:55:13,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741961_1137 (size=12301) 2024-11-25T05:55:13,827 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=698 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/696f388190f7477c8cdc048c30324f75 2024-11-25T05:55:13,847 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#C#compaction#125 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:13,847 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/83ca8e8b64c94fac86f325fbff07989b is 50, key is test_row_0/C:col10/1732514113053/Put/seqid=0 2024-11-25T05:55:13,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/3f5c1e7e77ae45ffaf2bd0ab00263f72 is 50, key is test_row_0/B:col10/1732514113083/Put/seqid=0 2024-11-25T05:55:13,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741962_1138 (size=13663) 2024-11-25T05:55:13,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741963_1139 (size=12301) 2024-11-25T05:55:13,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-25T05:55:13,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:13,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514173892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:13,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:13,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:13,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514173892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:13,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:13,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:13,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514173892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:13,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:13,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:13,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514173892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:13,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:13,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:13,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514173922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:13,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:13,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:14,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:14,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514174112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:14,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:14,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514174113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:14,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:14,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:14,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:14,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514174113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:14,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514174113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:14,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:14,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:14,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:14,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:14,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514174132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:14,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:14,206 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/6a9b34bdc2494a49a7a78085487baed0 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/6a9b34bdc2494a49a7a78085487baed0 2024-11-25T05:55:14,215 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/A of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into 6a9b34bdc2494a49a7a78085487baed0(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:14,215 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:14,215 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/A, priority=13, startTime=1732514113697; duration=0sec 2024-11-25T05:55:14,215 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:14,215 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:A 2024-11-25T05:55:14,274 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=698 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/3f5c1e7e77ae45ffaf2bd0ab00263f72 2024-11-25T05:55:14,287 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/83ca8e8b64c94fac86f325fbff07989b as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/83ca8e8b64c94fac86f325fbff07989b 2024-11-25T05:55:14,295 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/C of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into 83ca8e8b64c94fac86f325fbff07989b(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:14,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/befdf09ffbb840cdb7390c55ab7d16e6 is 50, key is test_row_0/C:col10/1732514113083/Put/seqid=0 2024-11-25T05:55:14,295 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:14,295 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/C, priority=13, startTime=1732514113697; duration=0sec 2024-11-25T05:55:14,295 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:14,295 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:C 2024-11-25T05:55:14,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741964_1140 (size=12301) 2024-11-25T05:55:14,311 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=698 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/befdf09ffbb840cdb7390c55ab7d16e6 2024-11-25T05:55:14,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/696f388190f7477c8cdc048c30324f75 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/696f388190f7477c8cdc048c30324f75 2024-11-25T05:55:14,323 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/696f388190f7477c8cdc048c30324f75, entries=150, sequenceid=698, filesize=12.0 K 2024-11-25T05:55:14,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/3f5c1e7e77ae45ffaf2bd0ab00263f72 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/3f5c1e7e77ae45ffaf2bd0ab00263f72 2024-11-25T05:55:14,332 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/3f5c1e7e77ae45ffaf2bd0ab00263f72, entries=150, sequenceid=698, filesize=12.0 K 2024-11-25T05:55:14,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/befdf09ffbb840cdb7390c55ab7d16e6 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/befdf09ffbb840cdb7390c55ab7d16e6 2024-11-25T05:55:14,349 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/befdf09ffbb840cdb7390c55ab7d16e6, entries=150, sequenceid=698, filesize=12.0 K 2024-11-25T05:55:14,351 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for cf8ce3f4f3ad5ac4b5297b0ca903cd34 in 632ms, sequenceid=698, compaction requested=false 2024-11-25T05:55:14,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:14,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:14,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-25T05:55:14,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-25T05:55:14,356 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-25T05:55:14,356 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0960 sec 2024-11-25T05:55:14,359 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.1040 sec 2024-11-25T05:55:14,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-25T05:55:14,392 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:55:14,393 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:55:14,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-25T05:55:14,395 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:55:14,396 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:55:14,396 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:55:14,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-25T05:55:14,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:55:14,425 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-25T05:55:14,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=A 2024-11-25T05:55:14,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:14,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=B 2024-11-25T05:55:14,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:14,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=C 2024-11-25T05:55:14,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:14,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/28a9742750e143a4b0b26c394cfce932 is 50, key is test_row_0/A:col10/1732514114424/Put/seqid=0 2024-11-25T05:55:14,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:14,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514174443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:14,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:14,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514174443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:14,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:14,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:14,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:14,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514174445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:14,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:14,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:14,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514174446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:14,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:14,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:14,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514174448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:14,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:14,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741965_1141 (size=17181) 2024-11-25T05:55:14,476 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=720 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/28a9742750e143a4b0b26c394cfce932 2024-11-25T05:55:14,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/bce56439e03447edb9df24d01215fa25 is 50, key is test_row_0/B:col10/1732514114424/Put/seqid=0 2024-11-25T05:55:14,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-25T05:55:14,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741966_1142 (size=12301) 2024-11-25T05:55:14,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-25T05:55:14,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:14,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:14,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:14,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:14,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:14,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:14,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:14,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514174552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:14,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:14,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:14,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514174563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:14,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:14,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:14,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514174563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:14,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:14,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:14,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514174563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:14,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:14,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-25T05:55:14,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:14,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:14,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:14,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:14,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:14,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:14,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-25T05:55:14,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:14,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47340 deadline: 1732514174772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:14,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:14,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47386 deadline: 1732514174772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:14,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:14,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:14,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47412 deadline: 1732514174772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:14,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:14,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:14,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:14,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47380 deadline: 1732514174772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:14,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-25T05:55:14,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:14,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:14,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:14,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:14,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:14,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:14,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=720 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/bce56439e03447edb9df24d01215fa25 2024-11-25T05:55:14,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/d649983eb9e84d9c80d3757f739af8d7 is 50, key is test_row_0/C:col10/1732514114424/Put/seqid=0 2024-11-25T05:55:14,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:14,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47366 deadline: 1732514174961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:14,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741967_1143 (size=12301) 2024-11-25T05:55:14,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=720 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/d649983eb9e84d9c80d3757f739af8d7 2024-11-25T05:55:15,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/28a9742750e143a4b0b26c394cfce932 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/28a9742750e143a4b0b26c394cfce932 2024-11-25T05:55:15,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/28a9742750e143a4b0b26c394cfce932, entries=250, sequenceid=720, filesize=16.8 K 2024-11-25T05:55:15,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/bce56439e03447edb9df24d01215fa25 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/bce56439e03447edb9df24d01215fa25 2024-11-25T05:55:15,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-25T05:55:15,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:15,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:15,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:15,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:15,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:15,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:15,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/bce56439e03447edb9df24d01215fa25, entries=150, sequenceid=720, filesize=12.0 K 2024-11-25T05:55:15,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/d649983eb9e84d9c80d3757f739af8d7 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/d649983eb9e84d9c80d3757f739af8d7 2024-11-25T05:55:15,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-25T05:55:15,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/d649983eb9e84d9c80d3757f739af8d7, entries=150, sequenceid=720, filesize=12.0 K 2024-11-25T05:55:15,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for cf8ce3f4f3ad5ac4b5297b0ca903cd34 in 606ms, sequenceid=720, compaction requested=true 2024-11-25T05:55:15,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:15,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:55:15,032 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:15,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:15,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:55:15,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:15,032 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:15,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cf8ce3f4f3ad5ac4b5297b0ca903cd34:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:55:15,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:15,033 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:15,033 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43179 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:15,033 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/B is initiating minor compaction (all files) 2024-11-25T05:55:15,033 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/A is initiating minor compaction (all files) 2024-11-25T05:55:15,033 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/B in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:15,034 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/A in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:15,034 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/6a9b34bdc2494a49a7a78085487baed0, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/696f388190f7477c8cdc048c30324f75, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/28a9742750e143a4b0b26c394cfce932] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=42.2 K 2024-11-25T05:55:15,034 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/b829be5d7ec1493db5f7ad49252af3dc, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/3f5c1e7e77ae45ffaf2bd0ab00263f72, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/bce56439e03447edb9df24d01215fa25] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=37.4 K 2024-11-25T05:55:15,035 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a9b34bdc2494a49a7a78085487baed0, keycount=150, bloomtype=ROW, size=13.4 K, encoding=NONE, compression=NONE, seqNum=682, earliestPutTs=1732514112408 2024-11-25T05:55:15,035 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting b829be5d7ec1493db5f7ad49252af3dc, keycount=150, bloomtype=ROW, size=13.4 K, encoding=NONE, compression=NONE, seqNum=682, earliestPutTs=1732514112408 2024-11-25T05:55:15,035 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 696f388190f7477c8cdc048c30324f75, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=698, earliestPutTs=1732514113072 2024-11-25T05:55:15,035 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f5c1e7e77ae45ffaf2bd0ab00263f72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=698, earliestPutTs=1732514113072 2024-11-25T05:55:15,039 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting bce56439e03447edb9df24d01215fa25, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=720, earliestPutTs=1732514113760 2024-11-25T05:55:15,039 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28a9742750e143a4b0b26c394cfce932, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=720, earliestPutTs=1732514113748 2024-11-25T05:55:15,054 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#A#compaction#131 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:15,054 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/c1375ad078d94d5d977f48ab9e0a0f21 is 50, key is test_row_0/A:col10/1732514114424/Put/seqid=0 2024-11-25T05:55:15,065 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#B#compaction#132 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:15,065 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/dfe8f92814c74607bcaa5e0a34d11ed8 is 50, key is test_row_0/B:col10/1732514114424/Put/seqid=0 2024-11-25T05:55:15,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741969_1145 (size=13799) 2024-11-25T05:55:15,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:55:15,086 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-25T05:55:15,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=A 2024-11-25T05:55:15,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:15,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=B 2024-11-25T05:55:15,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:15,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=C 2024-11-25T05:55:15,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:15,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741968_1144 (size=13799) 2024-11-25T05:55:15,093 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/dfe8f92814c74607bcaa5e0a34d11ed8 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/dfe8f92814c74607bcaa5e0a34d11ed8 2024-11-25T05:55:15,100 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/c1375ad078d94d5d977f48ab9e0a0f21 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/c1375ad078d94d5d977f48ab9e0a0f21 2024-11-25T05:55:15,101 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-25T05:55:15,101 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:55:15,102 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/B of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into dfe8f92814c74607bcaa5e0a34d11ed8(size=13.5 K), total size for store is 13.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:15,102 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:15,102 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/B, priority=13, startTime=1732514115032; duration=0sec 2024-11-25T05:55:15,102 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:15,102 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:B 2024-11-25T05:55:15,102 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:15,104 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:15,104 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): cf8ce3f4f3ad5ac4b5297b0ca903cd34/C is initiating minor compaction (all files) 2024-11-25T05:55:15,104 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cf8ce3f4f3ad5ac4b5297b0ca903cd34/C in TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:15,104 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/83ca8e8b64c94fac86f325fbff07989b, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/befdf09ffbb840cdb7390c55ab7d16e6, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/d649983eb9e84d9c80d3757f739af8d7] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp, totalSize=37.4 K 2024-11-25T05:55:15,104 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 83ca8e8b64c94fac86f325fbff07989b, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=682, earliestPutTs=1732514112408 2024-11-25T05:55:15,105 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting befdf09ffbb840cdb7390c55ab7d16e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=698, earliestPutTs=1732514113072 2024-11-25T05:55:15,106 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-25T05:55:15,106 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:55:15,107 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting d649983eb9e84d9c80d3757f739af8d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=720, earliestPutTs=1732514113760 2024-11-25T05:55:15,107 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-25T05:55:15,107 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:55:15,108 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:15,108 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:15,108 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-25T05:55:15,108 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:55:15,108 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:15,108 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/A of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into c1375ad078d94d5d977f48ab9e0a0f21(size=13.5 K), total size for store is 13.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:15,108 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:15,108 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/A, priority=13, startTime=1732514115031; duration=0sec 2024-11-25T05:55:15,108 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:15,108 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:15,108 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:15,108 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:A 2024-11-25T05:55:15,108 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:55:15,108 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:55:15,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/8514846fb379474a94ae77f86f88cbf5 is 50, key is test_row_0/A:col10/1732514114446/Put/seqid=0 2024-11-25T05:55:15,110 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-25T05:55:15,110 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:55:15,110 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:15,111 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:15,111 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:55:15,111 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:15,111 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:55:15,112 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:15,112 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:15,112 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:55:15,112 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-25T05:55:15,112 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:55:15,112 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:15,112 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:15,112 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:55:15,115 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-25T05:55:15,115 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:55:15,115 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:15,115 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:15,115 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:55:15,125 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-25T05:55:15,125 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:55:15,126 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:15,126 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:15,126 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:55:15,127 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cf8ce3f4f3ad5ac4b5297b0ca903cd34#C#compaction#134 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:15,128 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/2867882cf4d945e294be1c4a1a86bdfb is 50, key is test_row_0/C:col10/1732514114424/Put/seqid=0 2024-11-25T05:55:15,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741970_1146 (size=14741) 2024-11-25T05:55:15,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=734 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/8514846fb379474a94ae77f86f88cbf5 2024-11-25T05:55:15,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741971_1147 (size=13765) 2024-11-25T05:55:15,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/33947c0ba9824521b3f65897e8de63d5 is 50, key is test_row_0/B:col10/1732514114446/Put/seqid=0 2024-11-25T05:55:15,143 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/2867882cf4d945e294be1c4a1a86bdfb as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/2867882cf4d945e294be1c4a1a86bdfb 2024-11-25T05:55:15,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741972_1148 (size=12301) 2024-11-25T05:55:15,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=734 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/33947c0ba9824521b3f65897e8de63d5 2024-11-25T05:55:15,153 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cf8ce3f4f3ad5ac4b5297b0ca903cd34/C of cf8ce3f4f3ad5ac4b5297b0ca903cd34 into 2867882cf4d945e294be1c4a1a86bdfb(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:15,153 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:15,153 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., storeName=cf8ce3f4f3ad5ac4b5297b0ca903cd34/C, priority=13, startTime=1732514115032; duration=0sec 2024-11-25T05:55:15,153 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:15,153 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cf8ce3f4f3ad5ac4b5297b0ca903cd34:C 2024-11-25T05:55:15,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/8b06ce67b9a24d3ca0b9ad78a48d5bba is 50, key is test_row_0/C:col10/1732514114446/Put/seqid=0 2024-11-25T05:55:15,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741973_1149 (size=12301) 2024-11-25T05:55:15,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=734 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/8b06ce67b9a24d3ca0b9ad78a48d5bba 2024-11-25T05:55:15,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-25T05:55:15,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:15,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:15,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:15,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:15,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:15,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:15,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/8514846fb379474a94ae77f86f88cbf5 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/8514846fb379474a94ae77f86f88cbf5 2024-11-25T05:55:15,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/8514846fb379474a94ae77f86f88cbf5, entries=200, sequenceid=734, filesize=14.4 K 2024-11-25T05:55:15,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/33947c0ba9824521b3f65897e8de63d5 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/33947c0ba9824521b3f65897e8de63d5 2024-11-25T05:55:15,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/33947c0ba9824521b3f65897e8de63d5, entries=150, sequenceid=734, filesize=12.0 K 2024-11-25T05:55:15,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/8b06ce67b9a24d3ca0b9ad78a48d5bba as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/8b06ce67b9a24d3ca0b9ad78a48d5bba 2024-11-25T05:55:15,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/8b06ce67b9a24d3ca0b9ad78a48d5bba, entries=150, sequenceid=734, filesize=12.0 K 2024-11-25T05:55:15,194 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=40.25 KB/41220 for cf8ce3f4f3ad5ac4b5297b0ca903cd34 in 108ms, sequenceid=734, compaction requested=false 2024-11-25T05:55:15,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:15,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-25T05:55:15,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:15,321 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-25T05:55:15,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=A 2024-11-25T05:55:15,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:15,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=B 2024-11-25T05:55:15,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:15,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=C 2024-11-25T05:55:15,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:15,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/99ca73bc0d3e43a3a67ebdf4bd10d1bd is 50, key is test_row_0/A:col10/1732514115105/Put/seqid=0 2024-11-25T05:55:15,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741974_1150 (size=12301) 2024-11-25T05:55:15,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-25T05:55:15,731 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=746 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/99ca73bc0d3e43a3a67ebdf4bd10d1bd 2024-11-25T05:55:15,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/354047ed3b984dd68228ba90a204aee3 is 50, key is test_row_0/B:col10/1732514115105/Put/seqid=0 2024-11-25T05:55:15,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741975_1151 (size=12301) 2024-11-25T05:55:15,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:55:15,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. as already flushing 2024-11-25T05:55:15,984 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-25T05:55:15,984 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:55:15,984 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:15,984 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:15,984 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:55:16,144 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=746 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/354047ed3b984dd68228ba90a204aee3 2024-11-25T05:55:16,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/b2138007688d40039cabcd6a29f74ff9 is 50, key is test_row_0/C:col10/1732514115105/Put/seqid=0 2024-11-25T05:55:16,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741976_1152 (size=12301) 2024-11-25T05:55:16,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-25T05:55:16,559 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=746 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/b2138007688d40039cabcd6a29f74ff9 2024-11-25T05:55:16,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/99ca73bc0d3e43a3a67ebdf4bd10d1bd as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/99ca73bc0d3e43a3a67ebdf4bd10d1bd 2024-11-25T05:55:16,581 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/99ca73bc0d3e43a3a67ebdf4bd10d1bd, entries=150, sequenceid=746, filesize=12.0 K 2024-11-25T05:55:16,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/354047ed3b984dd68228ba90a204aee3 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/354047ed3b984dd68228ba90a204aee3 2024-11-25T05:55:16,588 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/354047ed3b984dd68228ba90a204aee3, entries=150, sequenceid=746, filesize=12.0 K 2024-11-25T05:55:16,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/b2138007688d40039cabcd6a29f74ff9 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/b2138007688d40039cabcd6a29f74ff9 2024-11-25T05:55:16,594 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/b2138007688d40039cabcd6a29f74ff9, entries=150, sequenceid=746, filesize=12.0 K 2024-11-25T05:55:16,595 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=6.71 KB/6870 for cf8ce3f4f3ad5ac4b5297b0ca903cd34 in 1274ms, sequenceid=746, compaction requested=true 2024-11-25T05:55:16,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: 2024-11-25T05:55:16,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:16,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-25T05:55:16,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-25T05:55:16,598 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-25T05:55:16,598 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2000 sec 2024-11-25T05:55:16,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.2060 sec 2024-11-25T05:55:17,544 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T05:55:18,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-25T05:55:18,572 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:55:18,572 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-25T05:55:18,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 127 2024-11-25T05:55:18,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 127 2024-11-25T05:55:18,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 113 2024-11-25T05:55:18,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 113 2024-11-25T05:55:18,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 126 2024-11-25T05:55:18,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-25T05:55:18,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1981 2024-11-25T05:55:18,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1970 2024-11-25T05:55:18,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-25T05:55:18,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 919 2024-11-25T05:55:18,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2748 rows 2024-11-25T05:55:18,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 918 2024-11-25T05:55:18,573 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2751 rows 2024-11-25T05:55:18,574 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T05:55:18,574 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T05:55:18,574 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:18,574 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:18,574 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:55:18,580 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-25T05:55:18,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-25T05:55:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-25T05:55:18,592 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732514118591"}]},"ts":"1732514118591"} 2024-11-25T05:55:18,595 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-25T05:55:18,595 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-25T05:55:18,598 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T05:55:18,605 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf8ce3f4f3ad5ac4b5297b0ca903cd34, UNASSIGN}] 2024-11-25T05:55:18,607 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf8ce3f4f3ad5ac4b5297b0ca903cd34, UNASSIGN 2024-11-25T05:55:18,609 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=cf8ce3f4f3ad5ac4b5297b0ca903cd34, regionState=CLOSING, regionLocation=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:18,613 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf8ce3f4f3ad5ac4b5297b0ca903cd34, UNASSIGN because future has completed 2024-11-25T05:55:18,614 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T05:55:18,614 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005}] 2024-11-25T05:55:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-25T05:55:18,773 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:55:18,774 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T05:55:18,775 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing cf8ce3f4f3ad5ac4b5297b0ca903cd34, disabling compactions & flushes 2024-11-25T05:55:18,775 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:18,775 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:18,775 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. after waiting 0 ms 2024-11-25T05:55:18,775 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:18,775 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34 3/3 column families, dataSize=6.71 KB heapSize=20.67 KB 2024-11-25T05:55:18,775 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=A 2024-11-25T05:55:18,775 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:18,776 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=B 2024-11-25T05:55:18,776 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:18,776 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cf8ce3f4f3ad5ac4b5297b0ca903cd34, store=C 2024-11-25T05:55:18,776 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:18,782 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/d74ea2d1c4bb4dcab6b3295eb7a45161 is 50, key is test_row_1/A:col10/1732514115982/Put/seqid=0 2024-11-25T05:55:18,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741977_1153 (size=7415) 2024-11-25T05:55:18,793 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=750 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/d74ea2d1c4bb4dcab6b3295eb7a45161 2024-11-25T05:55:18,802 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/4b076b4611ce46bd8b644663c90971bc is 50, key is test_row_1/B:col10/1732514115982/Put/seqid=0 2024-11-25T05:55:18,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741978_1154 (size=7415) 2024-11-25T05:55:18,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-25T05:55:19,211 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=750 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/4b076b4611ce46bd8b644663c90971bc 2024-11-25T05:55:19,219 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/4e3a6645201f4eb2b8a75b24e5c9ff86 is 50, key is test_row_1/C:col10/1732514115982/Put/seqid=0 2024-11-25T05:55:19,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-25T05:55:19,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741979_1155 (size=7415) 2024-11-25T05:55:19,624 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=750 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/4e3a6645201f4eb2b8a75b24e5c9ff86 2024-11-25T05:55:19,630 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/A/d74ea2d1c4bb4dcab6b3295eb7a45161 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/d74ea2d1c4bb4dcab6b3295eb7a45161 2024-11-25T05:55:19,634 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/d74ea2d1c4bb4dcab6b3295eb7a45161, entries=50, sequenceid=750, filesize=7.2 K 2024-11-25T05:55:19,636 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/B/4b076b4611ce46bd8b644663c90971bc as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/4b076b4611ce46bd8b644663c90971bc 2024-11-25T05:55:19,641 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/4b076b4611ce46bd8b644663c90971bc, entries=50, sequenceid=750, filesize=7.2 K 2024-11-25T05:55:19,642 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/.tmp/C/4e3a6645201f4eb2b8a75b24e5c9ff86 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/4e3a6645201f4eb2b8a75b24e5c9ff86 2024-11-25T05:55:19,646 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/4e3a6645201f4eb2b8a75b24e5c9ff86, entries=50, sequenceid=750, filesize=7.2 K 2024-11-25T05:55:19,647 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for cf8ce3f4f3ad5ac4b5297b0ca903cd34 in 872ms, sequenceid=750, compaction requested=true 2024-11-25T05:55:19,648 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/3246d71eaa544a84af4849a86ce4a65c, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/f811aebd015942ebb437e971dd0ba385, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/bd9771db82994a15b536a0ccf907d1be, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/ee565dbed7e74bb89fc5604e76bfc8c4, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/4cfed5fe79764b748be44d695976755f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/2779c5bef419405bafc6deb24302fade, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/f0c6fc1812d34dd8ae8c1322d28fe729, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/65f8e7ee43134361919bedb6f71ec2cd, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/c39e8bb7b4bb415eb04df4e3b5e035e0, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/d755bf3560254e28833cbe8f8f216aba, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/0d72988bf0404061b94b7be810b8a473, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/7dbbcadfea1b4233ab46cf36e22e911e, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/7cce169531cc478bab68412d7ca44456, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/4849b1b74f084ae1aa99fbd605ca2b02, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/a3c1a33264dc4388be036a7ecd7546a6, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/06064228fdbe487d87a70c3b4cebbed2, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/febe19c0d4774f8aa30060125b68decf, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/d79f9f0eafa940b984d7165c3c626cad, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/4a07ef751a6141fc8681ca0836caa6e2, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/1883ff8e3bbc439ebc31c74e304527f2, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/d02c8391e19d408b99053bfece220ba8, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/019b2688c576421eb675d3f83d6da1b0, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/cd77d6986b124564914e8449cace7a8d, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/2086a1af63c34adca14f3c3230bd9050, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/20da8daddde54b578bcbc1194295d682, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/80b0a231452446e2b5138a0368f038f7, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/61b5b9fb1fa44761a83a4b25ddd3afd8, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/c702903f5230430995613720e614581b, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/0e18801864004a17b71bbea64afe0ee2, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/fbaed385a5e14e3bad4fc4fdd9404ea9, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/e8dbab9aacac45a6af913230c98888ba, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/61de2ff979bc491b84cc298079835f2f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/79a51fa6fc214414ac6c1ce9de9bac0c, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/4aca44e51c7c4f67aec1d74c77327f64, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/ca1255e514fc4cd59cc216374d25cfea, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/1aa915c43d2144c0b0b5f069dd817eb1, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/d993fee514df43ef951abd5e3bacfd20, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/aa42e43dd4064cea98bde5417faf425f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/85a4122fa7df4236bb70a0553f678291, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/41725bc2863f48a8bcadc164237a0543, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/659b7bdb5c944fe19786af6dae75c668, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/6a9b34bdc2494a49a7a78085487baed0, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/696f388190f7477c8cdc048c30324f75, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/28a9742750e143a4b0b26c394cfce932] to archive 2024-11-25T05:55:19,652 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T05:55:19,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/3246d71eaa544a84af4849a86ce4a65c to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/3246d71eaa544a84af4849a86ce4a65c 2024-11-25T05:55:19,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/f811aebd015942ebb437e971dd0ba385 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/f811aebd015942ebb437e971dd0ba385 2024-11-25T05:55:19,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/bd9771db82994a15b536a0ccf907d1be to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/bd9771db82994a15b536a0ccf907d1be 2024-11-25T05:55:19,665 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/ee565dbed7e74bb89fc5604e76bfc8c4 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/ee565dbed7e74bb89fc5604e76bfc8c4 2024-11-25T05:55:19,667 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/4cfed5fe79764b748be44d695976755f to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/4cfed5fe79764b748be44d695976755f 2024-11-25T05:55:19,670 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/2779c5bef419405bafc6deb24302fade to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/2779c5bef419405bafc6deb24302fade 2024-11-25T05:55:19,672 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/f0c6fc1812d34dd8ae8c1322d28fe729 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/f0c6fc1812d34dd8ae8c1322d28fe729 2024-11-25T05:55:19,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/65f8e7ee43134361919bedb6f71ec2cd to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/65f8e7ee43134361919bedb6f71ec2cd 2024-11-25T05:55:19,676 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/c39e8bb7b4bb415eb04df4e3b5e035e0 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/c39e8bb7b4bb415eb04df4e3b5e035e0 2024-11-25T05:55:19,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/d755bf3560254e28833cbe8f8f216aba to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/d755bf3560254e28833cbe8f8f216aba 2024-11-25T05:55:19,680 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/0d72988bf0404061b94b7be810b8a473 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/0d72988bf0404061b94b7be810b8a473 2024-11-25T05:55:19,682 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/7dbbcadfea1b4233ab46cf36e22e911e to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/7dbbcadfea1b4233ab46cf36e22e911e 2024-11-25T05:55:19,683 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/7cce169531cc478bab68412d7ca44456 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/7cce169531cc478bab68412d7ca44456 2024-11-25T05:55:19,685 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/4849b1b74f084ae1aa99fbd605ca2b02 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/4849b1b74f084ae1aa99fbd605ca2b02 2024-11-25T05:55:19,687 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/a3c1a33264dc4388be036a7ecd7546a6 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/a3c1a33264dc4388be036a7ecd7546a6 2024-11-25T05:55:19,688 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/06064228fdbe487d87a70c3b4cebbed2 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/06064228fdbe487d87a70c3b4cebbed2 2024-11-25T05:55:19,690 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/febe19c0d4774f8aa30060125b68decf to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/febe19c0d4774f8aa30060125b68decf 2024-11-25T05:55:19,692 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/d79f9f0eafa940b984d7165c3c626cad to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/d79f9f0eafa940b984d7165c3c626cad 2024-11-25T05:55:19,694 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/4a07ef751a6141fc8681ca0836caa6e2 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/4a07ef751a6141fc8681ca0836caa6e2 2024-11-25T05:55:19,696 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/1883ff8e3bbc439ebc31c74e304527f2 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/1883ff8e3bbc439ebc31c74e304527f2 2024-11-25T05:55:19,698 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/d02c8391e19d408b99053bfece220ba8 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/d02c8391e19d408b99053bfece220ba8 2024-11-25T05:55:19,700 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/019b2688c576421eb675d3f83d6da1b0 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/019b2688c576421eb675d3f83d6da1b0 2024-11-25T05:55:19,702 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/cd77d6986b124564914e8449cace7a8d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/cd77d6986b124564914e8449cace7a8d 2024-11-25T05:55:19,704 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/2086a1af63c34adca14f3c3230bd9050 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/2086a1af63c34adca14f3c3230bd9050 2024-11-25T05:55:19,707 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/20da8daddde54b578bcbc1194295d682 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/20da8daddde54b578bcbc1194295d682 2024-11-25T05:55:19,709 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/80b0a231452446e2b5138a0368f038f7 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/80b0a231452446e2b5138a0368f038f7 2024-11-25T05:55:19,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/61b5b9fb1fa44761a83a4b25ddd3afd8 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/61b5b9fb1fa44761a83a4b25ddd3afd8 2024-11-25T05:55:19,714 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/c702903f5230430995613720e614581b to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/c702903f5230430995613720e614581b 2024-11-25T05:55:19,716 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/0e18801864004a17b71bbea64afe0ee2 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/0e18801864004a17b71bbea64afe0ee2 2024-11-25T05:55:19,718 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/fbaed385a5e14e3bad4fc4fdd9404ea9 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/fbaed385a5e14e3bad4fc4fdd9404ea9 2024-11-25T05:55:19,720 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/e8dbab9aacac45a6af913230c98888ba to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/e8dbab9aacac45a6af913230c98888ba 2024-11-25T05:55:19,723 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/61de2ff979bc491b84cc298079835f2f to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/61de2ff979bc491b84cc298079835f2f 2024-11-25T05:55:19,725 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/79a51fa6fc214414ac6c1ce9de9bac0c to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/79a51fa6fc214414ac6c1ce9de9bac0c 2024-11-25T05:55:19,727 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/4aca44e51c7c4f67aec1d74c77327f64 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/4aca44e51c7c4f67aec1d74c77327f64 2024-11-25T05:55:19,730 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/ca1255e514fc4cd59cc216374d25cfea to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/ca1255e514fc4cd59cc216374d25cfea 2024-11-25T05:55:19,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-25T05:55:19,733 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/1aa915c43d2144c0b0b5f069dd817eb1 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/1aa915c43d2144c0b0b5f069dd817eb1 2024-11-25T05:55:19,736 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/d993fee514df43ef951abd5e3bacfd20 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/d993fee514df43ef951abd5e3bacfd20 2024-11-25T05:55:19,738 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/aa42e43dd4064cea98bde5417faf425f to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/aa42e43dd4064cea98bde5417faf425f 2024-11-25T05:55:19,741 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/85a4122fa7df4236bb70a0553f678291 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/85a4122fa7df4236bb70a0553f678291 2024-11-25T05:55:19,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/41725bc2863f48a8bcadc164237a0543 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/41725bc2863f48a8bcadc164237a0543 2024-11-25T05:55:19,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/659b7bdb5c944fe19786af6dae75c668 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/659b7bdb5c944fe19786af6dae75c668 2024-11-25T05:55:19,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/6a9b34bdc2494a49a7a78085487baed0 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/6a9b34bdc2494a49a7a78085487baed0 2024-11-25T05:55:19,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/696f388190f7477c8cdc048c30324f75 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/696f388190f7477c8cdc048c30324f75 2024-11-25T05:55:19,753 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/28a9742750e143a4b0b26c394cfce932 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/28a9742750e143a4b0b26c394cfce932 2024-11-25T05:55:19,772 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/6d6bdc853a7d487ea551c72ca4d240ae, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/123037e539c8470887975e967cb1b0c3, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/b266dedc818e429abe71873f430ee80b, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/5eabb2c3f6db41018dfea8bec92d1f86, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/e66baecc70384ea08d537ff7fa87f5da, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/77d09ef194e041c7b981ae3b8975ff17, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/6d06f52b20dc4599a2f9438526abce10, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/af6102ff989745dca5794d9746b1176c, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/5100763999c940368b315970163e446a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/71709038fb994bd5acbd9a5669455102, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/cfa41c4410cf48dc97a4a2cbb507fb85, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/3cd5b80953fe4c54a7cf822f15d4dfdc, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/bb3ccdee8b344b43bd6826ad248f0702, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/6cc728eafdb1499eb102e9cb98e20479, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/d851c78e5e4f40cd89bc3a3d5e0a4f3a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/bbba73e8e4fa47a3b822dd3b0605b90d, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/7e40c6c31b7745b6acefd3afda962686, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/e8f22d60223443cab44f3e15b88a5e15, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/85c3b3d45b9942648f03a82875290b96, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/4bea555fd78e4741b78b05b18032d5f3, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/e84ab1ad464545aa853d71d7456949dd, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/bccf2a3721eb4512a0ed3be82bd208e7, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/5c1c949d87cb4ea0821f1de093be05e8, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/d13a2e69eb2e4a4da160228f4a95ccca, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/7ed88332848b41159f064f0874b03115, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/7fe7cae59ee145fdac968a7b9d7767c5, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/bdc10dfc947f45d5a0a62bb816989fe7, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/072b55ca7e6048f2a95f77f521b0c840, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/582fb101d7ce4c04bc41bdbfef2ed6f6, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/865741f1d5e44b0a8a26797cdefd571d, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/ffabb5839067498b8e2837fc1f3bff4d, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/7ad83bd490bc471c84f518d3df630d08, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/7b280806097a432d922d12e1ccf1df6a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/f247339faffe4d65920953c7bb1f2d84, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/3068d9223dde48b3929434f688953ca4, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/60d039bfa017403f9d61879227ea194a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/1932ae9dbdc7460a984e8fc0a3f9a882, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/2809d3d167aa45889dfe80f3d54c9a32, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/b40b12b0f65f411eadfdd143111dafaa, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/c5ef8fc34fab4e519d627b223e028e6e, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/b829be5d7ec1493db5f7ad49252af3dc, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/cf3246d2542547aba6e302564ccc68d9, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/3f5c1e7e77ae45ffaf2bd0ab00263f72, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/bce56439e03447edb9df24d01215fa25] to archive 2024-11-25T05:55:19,773 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T05:55:19,776 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/6d6bdc853a7d487ea551c72ca4d240ae to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/6d6bdc853a7d487ea551c72ca4d240ae 2024-11-25T05:55:19,777 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/123037e539c8470887975e967cb1b0c3 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/123037e539c8470887975e967cb1b0c3 2024-11-25T05:55:19,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/b266dedc818e429abe71873f430ee80b to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/b266dedc818e429abe71873f430ee80b 2024-11-25T05:55:19,781 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/5eabb2c3f6db41018dfea8bec92d1f86 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/5eabb2c3f6db41018dfea8bec92d1f86 2024-11-25T05:55:19,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/e66baecc70384ea08d537ff7fa87f5da to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/e66baecc70384ea08d537ff7fa87f5da 2024-11-25T05:55:19,785 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/77d09ef194e041c7b981ae3b8975ff17 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/77d09ef194e041c7b981ae3b8975ff17 2024-11-25T05:55:19,787 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/6d06f52b20dc4599a2f9438526abce10 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/6d06f52b20dc4599a2f9438526abce10 2024-11-25T05:55:19,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/af6102ff989745dca5794d9746b1176c to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/af6102ff989745dca5794d9746b1176c 2024-11-25T05:55:19,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/5100763999c940368b315970163e446a to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/5100763999c940368b315970163e446a 2024-11-25T05:55:19,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/71709038fb994bd5acbd9a5669455102 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/71709038fb994bd5acbd9a5669455102 2024-11-25T05:55:19,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/cfa41c4410cf48dc97a4a2cbb507fb85 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/cfa41c4410cf48dc97a4a2cbb507fb85 2024-11-25T05:55:19,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/3cd5b80953fe4c54a7cf822f15d4dfdc to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/3cd5b80953fe4c54a7cf822f15d4dfdc 2024-11-25T05:55:19,794 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/bb3ccdee8b344b43bd6826ad248f0702 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/bb3ccdee8b344b43bd6826ad248f0702 2024-11-25T05:55:19,795 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/6cc728eafdb1499eb102e9cb98e20479 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/6cc728eafdb1499eb102e9cb98e20479 2024-11-25T05:55:19,796 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/d851c78e5e4f40cd89bc3a3d5e0a4f3a to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/d851c78e5e4f40cd89bc3a3d5e0a4f3a 2024-11-25T05:55:19,797 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/bbba73e8e4fa47a3b822dd3b0605b90d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/bbba73e8e4fa47a3b822dd3b0605b90d 2024-11-25T05:55:19,799 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/7e40c6c31b7745b6acefd3afda962686 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/7e40c6c31b7745b6acefd3afda962686 2024-11-25T05:55:19,800 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/e8f22d60223443cab44f3e15b88a5e15 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/e8f22d60223443cab44f3e15b88a5e15 2024-11-25T05:55:19,801 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/85c3b3d45b9942648f03a82875290b96 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/85c3b3d45b9942648f03a82875290b96 2024-11-25T05:55:19,802 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/4bea555fd78e4741b78b05b18032d5f3 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/4bea555fd78e4741b78b05b18032d5f3 2024-11-25T05:55:19,804 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/e84ab1ad464545aa853d71d7456949dd to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/e84ab1ad464545aa853d71d7456949dd 2024-11-25T05:55:19,805 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/bccf2a3721eb4512a0ed3be82bd208e7 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/bccf2a3721eb4512a0ed3be82bd208e7 2024-11-25T05:55:19,806 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/5c1c949d87cb4ea0821f1de093be05e8 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/5c1c949d87cb4ea0821f1de093be05e8 2024-11-25T05:55:19,807 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/d13a2e69eb2e4a4da160228f4a95ccca to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/d13a2e69eb2e4a4da160228f4a95ccca 2024-11-25T05:55:19,808 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/7ed88332848b41159f064f0874b03115 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/7ed88332848b41159f064f0874b03115 2024-11-25T05:55:19,810 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/7fe7cae59ee145fdac968a7b9d7767c5 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/7fe7cae59ee145fdac968a7b9d7767c5 2024-11-25T05:55:19,811 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/bdc10dfc947f45d5a0a62bb816989fe7 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/bdc10dfc947f45d5a0a62bb816989fe7 2024-11-25T05:55:19,812 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/072b55ca7e6048f2a95f77f521b0c840 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/072b55ca7e6048f2a95f77f521b0c840 2024-11-25T05:55:19,813 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/582fb101d7ce4c04bc41bdbfef2ed6f6 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/582fb101d7ce4c04bc41bdbfef2ed6f6 2024-11-25T05:55:19,814 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/865741f1d5e44b0a8a26797cdefd571d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/865741f1d5e44b0a8a26797cdefd571d 2024-11-25T05:55:19,816 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/ffabb5839067498b8e2837fc1f3bff4d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/ffabb5839067498b8e2837fc1f3bff4d 2024-11-25T05:55:19,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/7ad83bd490bc471c84f518d3df630d08 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/7ad83bd490bc471c84f518d3df630d08 2024-11-25T05:55:19,818 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/7b280806097a432d922d12e1ccf1df6a to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/7b280806097a432d922d12e1ccf1df6a 2024-11-25T05:55:19,819 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/f247339faffe4d65920953c7bb1f2d84 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/f247339faffe4d65920953c7bb1f2d84 2024-11-25T05:55:19,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/3068d9223dde48b3929434f688953ca4 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/3068d9223dde48b3929434f688953ca4 2024-11-25T05:55:19,821 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/60d039bfa017403f9d61879227ea194a to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/60d039bfa017403f9d61879227ea194a 2024-11-25T05:55:19,822 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/1932ae9dbdc7460a984e8fc0a3f9a882 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/1932ae9dbdc7460a984e8fc0a3f9a882 2024-11-25T05:55:19,823 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/2809d3d167aa45889dfe80f3d54c9a32 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/2809d3d167aa45889dfe80f3d54c9a32 2024-11-25T05:55:19,824 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/b40b12b0f65f411eadfdd143111dafaa to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/b40b12b0f65f411eadfdd143111dafaa 2024-11-25T05:55:19,825 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/c5ef8fc34fab4e519d627b223e028e6e to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/c5ef8fc34fab4e519d627b223e028e6e 2024-11-25T05:55:19,827 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/b829be5d7ec1493db5f7ad49252af3dc to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/b829be5d7ec1493db5f7ad49252af3dc 2024-11-25T05:55:19,828 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/cf3246d2542547aba6e302564ccc68d9 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/cf3246d2542547aba6e302564ccc68d9 2024-11-25T05:55:19,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/3f5c1e7e77ae45ffaf2bd0ab00263f72 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/3f5c1e7e77ae45ffaf2bd0ab00263f72 2024-11-25T05:55:19,830 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/bce56439e03447edb9df24d01215fa25 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/bce56439e03447edb9df24d01215fa25 2024-11-25T05:55:19,832 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/7abcf038322249a0b81f5e3fca326f2a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/339ad02410d5431a816bf3e60061afbc, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/2869d05cfca94ac3ad84535cc9ae9c8a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/8520dae074d2462fa9fc8da3391e67f8, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/a3055e98768f4aefbba68629f83a2585, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/b262418a9add470db153964485b3ffd7, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/0dd70a2a3a164cb9bdff950b529a696a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/0d217429f2da4e598df28a86cfd4847e, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/4374630a9b1348feada0fb5a03f02317, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/981c73fee7a84a1f907c69065d5d110e, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/65fd1217fd81459b9e5ed62d153bd0a7, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/a522bd8b97bf4f598f9fd6586007f598, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/1447f666f6de4f2d84584b89ae5181ad, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/696b6c48962c4954a73aa38f35bc3929, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/39b1bcab2759400284c2fd359d0c4593, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/f9d0cce73503405aab2c12032ca7207e, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/ca5604e9287540e198edd05fd0bef9f3, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/9ee0e1b62776466bacc959514a3f2b6b, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/c9030d10f7bb4138a21480061df66bb5, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/9682072354ec4edaa97962198b6318f5, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/a9f43efffc134fefb943202038188b19, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/901be871432244649e76501a29e2e3ba, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/06a24c6f5e554b4a86a91652936cdac9, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/d4b15ed789c440ab9c9c46ee3cc6bfbe, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/642c20ea3e7e4a69b7641e4630b8912a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/81bde786ad2c435da00aee08ac9b5b52, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/6a4b3abbaab3403bbd7df5d64bca60e7, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/c9c5171dded440a7960130d43b85df98, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/808f102497eb4e798adbc6dc2cda76ce, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/dc1166b8799f48a0bf16ab40a22fc971, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/5aba07e9c90a428b8f25d7013d2abf6e, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/7b5d6ec9962a412e93ae818cf8b0f278, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/d225455dd4dc4e52a0e0f50403c1ceec, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/944ee2589adf45b3a8b5662fd9975ae5, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/24f36679b0154aff9a2f07f8b031a706, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/6e204afd32544324b1a5cc157d33855c, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/6e79f5038205497c9c2177cbc0556854, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/97b5c25ea13f43e2a78196e3d7b6dc50, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/81f029697c0d475bb395f6b4d32676b6, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/83ca8e8b64c94fac86f325fbff07989b, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/89396391b4c941c799ad49fcd6919093, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/befdf09ffbb840cdb7390c55ab7d16e6, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/d649983eb9e84d9c80d3757f739af8d7] to archive 2024-11-25T05:55:19,833 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T05:55:19,835 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/7abcf038322249a0b81f5e3fca326f2a to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/7abcf038322249a0b81f5e3fca326f2a 2024-11-25T05:55:19,836 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/339ad02410d5431a816bf3e60061afbc to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/339ad02410d5431a816bf3e60061afbc 2024-11-25T05:55:19,837 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/2869d05cfca94ac3ad84535cc9ae9c8a to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/2869d05cfca94ac3ad84535cc9ae9c8a 2024-11-25T05:55:19,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/8520dae074d2462fa9fc8da3391e67f8 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/8520dae074d2462fa9fc8da3391e67f8 2024-11-25T05:55:19,840 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/a3055e98768f4aefbba68629f83a2585 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/a3055e98768f4aefbba68629f83a2585 2024-11-25T05:55:19,841 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/b262418a9add470db153964485b3ffd7 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/b262418a9add470db153964485b3ffd7 2024-11-25T05:55:19,842 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/0dd70a2a3a164cb9bdff950b529a696a to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/0dd70a2a3a164cb9bdff950b529a696a 2024-11-25T05:55:19,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/0d217429f2da4e598df28a86cfd4847e to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/0d217429f2da4e598df28a86cfd4847e 2024-11-25T05:55:19,844 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/4374630a9b1348feada0fb5a03f02317 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/4374630a9b1348feada0fb5a03f02317 2024-11-25T05:55:19,845 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/981c73fee7a84a1f907c69065d5d110e to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/981c73fee7a84a1f907c69065d5d110e 2024-11-25T05:55:19,846 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/65fd1217fd81459b9e5ed62d153bd0a7 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/65fd1217fd81459b9e5ed62d153bd0a7 2024-11-25T05:55:19,847 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/a522bd8b97bf4f598f9fd6586007f598 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/a522bd8b97bf4f598f9fd6586007f598 2024-11-25T05:55:19,848 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/1447f666f6de4f2d84584b89ae5181ad to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/1447f666f6de4f2d84584b89ae5181ad 2024-11-25T05:55:19,850 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/696b6c48962c4954a73aa38f35bc3929 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/696b6c48962c4954a73aa38f35bc3929 2024-11-25T05:55:19,851 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/39b1bcab2759400284c2fd359d0c4593 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/39b1bcab2759400284c2fd359d0c4593 2024-11-25T05:55:19,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/f9d0cce73503405aab2c12032ca7207e to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/f9d0cce73503405aab2c12032ca7207e 2024-11-25T05:55:19,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/ca5604e9287540e198edd05fd0bef9f3 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/ca5604e9287540e198edd05fd0bef9f3 2024-11-25T05:55:19,854 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/9ee0e1b62776466bacc959514a3f2b6b to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/9ee0e1b62776466bacc959514a3f2b6b 2024-11-25T05:55:19,856 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/c9030d10f7bb4138a21480061df66bb5 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/c9030d10f7bb4138a21480061df66bb5 2024-11-25T05:55:19,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/9682072354ec4edaa97962198b6318f5 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/9682072354ec4edaa97962198b6318f5 2024-11-25T05:55:19,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/a9f43efffc134fefb943202038188b19 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/a9f43efffc134fefb943202038188b19 2024-11-25T05:55:19,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/901be871432244649e76501a29e2e3ba to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/901be871432244649e76501a29e2e3ba 2024-11-25T05:55:19,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/06a24c6f5e554b4a86a91652936cdac9 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/06a24c6f5e554b4a86a91652936cdac9 2024-11-25T05:55:19,862 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/d4b15ed789c440ab9c9c46ee3cc6bfbe to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/d4b15ed789c440ab9c9c46ee3cc6bfbe 2024-11-25T05:55:19,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/642c20ea3e7e4a69b7641e4630b8912a to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/642c20ea3e7e4a69b7641e4630b8912a 2024-11-25T05:55:19,864 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/81bde786ad2c435da00aee08ac9b5b52 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/81bde786ad2c435da00aee08ac9b5b52 2024-11-25T05:55:19,865 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/6a4b3abbaab3403bbd7df5d64bca60e7 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/6a4b3abbaab3403bbd7df5d64bca60e7 2024-11-25T05:55:19,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/c9c5171dded440a7960130d43b85df98 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/c9c5171dded440a7960130d43b85df98 2024-11-25T05:55:19,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/808f102497eb4e798adbc6dc2cda76ce to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/808f102497eb4e798adbc6dc2cda76ce 2024-11-25T05:55:19,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/dc1166b8799f48a0bf16ab40a22fc971 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/dc1166b8799f48a0bf16ab40a22fc971 2024-11-25T05:55:19,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/5aba07e9c90a428b8f25d7013d2abf6e to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/5aba07e9c90a428b8f25d7013d2abf6e 2024-11-25T05:55:19,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/7b5d6ec9962a412e93ae818cf8b0f278 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/7b5d6ec9962a412e93ae818cf8b0f278 2024-11-25T05:55:19,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/d225455dd4dc4e52a0e0f50403c1ceec to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/d225455dd4dc4e52a0e0f50403c1ceec 2024-11-25T05:55:19,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/944ee2589adf45b3a8b5662fd9975ae5 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/944ee2589adf45b3a8b5662fd9975ae5 2024-11-25T05:55:19,875 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/24f36679b0154aff9a2f07f8b031a706 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/24f36679b0154aff9a2f07f8b031a706 2024-11-25T05:55:19,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/6e204afd32544324b1a5cc157d33855c to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/6e204afd32544324b1a5cc157d33855c 2024-11-25T05:55:19,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/6e79f5038205497c9c2177cbc0556854 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/6e79f5038205497c9c2177cbc0556854 2024-11-25T05:55:19,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/97b5c25ea13f43e2a78196e3d7b6dc50 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/97b5c25ea13f43e2a78196e3d7b6dc50 2024-11-25T05:55:19,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/81f029697c0d475bb395f6b4d32676b6 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/81f029697c0d475bb395f6b4d32676b6 2024-11-25T05:55:19,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/83ca8e8b64c94fac86f325fbff07989b to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/83ca8e8b64c94fac86f325fbff07989b 2024-11-25T05:55:19,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/89396391b4c941c799ad49fcd6919093 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/89396391b4c941c799ad49fcd6919093 2024-11-25T05:55:19,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/befdf09ffbb840cdb7390c55ab7d16e6 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/befdf09ffbb840cdb7390c55ab7d16e6 2024-11-25T05:55:19,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/d649983eb9e84d9c80d3757f739af8d7 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/d649983eb9e84d9c80d3757f739af8d7 2024-11-25T05:55:19,891 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/recovered.edits/753.seqid, newMaxSeqId=753, maxSeqId=1 2024-11-25T05:55:19,892 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. 2024-11-25T05:55:19,892 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for cf8ce3f4f3ad5ac4b5297b0ca903cd34: Waiting for close lock at 1732514118774Running coprocessor pre-close hooks at 1732514118774Disabling compacts and flushes for region at 1732514118774Disabling writes for close at 1732514118775 (+1 ms)Obtaining lock to block concurrent updates at 1732514118775Preparing flush snapshotting stores in cf8ce3f4f3ad5ac4b5297b0ca903cd34 at 1732514118775Finished memstore snapshotting TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=21120, getOffHeapSize=0, getCellsCount=150 at 1732514118776 (+1 ms)Flushing stores of TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34. at 1732514118776Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34/A: creating writer at 1732514118776Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34/A: appending metadata at 1732514118782 (+6 ms)Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34/A: closing flushed file at 1732514118782Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34/B: creating writer at 1732514118799 (+17 ms)Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34/B: appending metadata at 1732514118801 (+2 ms)Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34/B: closing flushed file at 1732514118802 (+1 ms)Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34/C: creating writer at 1732514119216 (+414 ms)Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34/C: appending metadata at 1732514119218 (+2 ms)Flushing cf8ce3f4f3ad5ac4b5297b0ca903cd34/C: closing flushed file at 1732514119219 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25d0806f: reopening flushed file at 1732514119629 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a951e41: reopening flushed file at 1732514119635 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68a43739: reopening flushed file at 1732514119641 (+6 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for cf8ce3f4f3ad5ac4b5297b0ca903cd34 in 872ms, sequenceid=750, compaction requested=true at 1732514119647 (+6 ms)Writing region close event to WAL at 1732514119887 (+240 ms)Running coprocessor post-close hooks at 1732514119892 (+5 ms)Closed at 1732514119892 2024-11-25T05:55:19,894 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:55:19,895 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=cf8ce3f4f3ad5ac4b5297b0ca903cd34, regionState=CLOSED 2024-11-25T05:55:19,897 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 because future has completed 2024-11-25T05:55:19,900 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-25T05:55:19,900 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure cf8ce3f4f3ad5ac4b5297b0ca903cd34, server=8ef925b832e3,35599,1732514089005 in 1.2840 sec 2024-11-25T05:55:19,902 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-25T05:55:19,902 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cf8ce3f4f3ad5ac4b5297b0ca903cd34, UNASSIGN in 1.2950 sec 2024-11-25T05:55:19,905 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-25T05:55:19,905 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.3050 sec 2024-11-25T05:55:19,906 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732514119906"}]},"ts":"1732514119906"} 2024-11-25T05:55:19,908 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-25T05:55:19,908 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-25T05:55:19,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.3270 sec 2024-11-25T05:55:20,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-25T05:55:20,742 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:55:20,745 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-25T05:55:20,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T05:55:20,749 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T05:55:20,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-25T05:55:20,751 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T05:55:20,756 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:55:20,760 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A, FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B, FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C, FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/recovered.edits] 2024-11-25T05:55:20,763 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/8514846fb379474a94ae77f86f88cbf5 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/8514846fb379474a94ae77f86f88cbf5 2024-11-25T05:55:20,764 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/99ca73bc0d3e43a3a67ebdf4bd10d1bd to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/99ca73bc0d3e43a3a67ebdf4bd10d1bd 2024-11-25T05:55:20,765 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/c1375ad078d94d5d977f48ab9e0a0f21 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/c1375ad078d94d5d977f48ab9e0a0f21 2024-11-25T05:55:20,766 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/d74ea2d1c4bb4dcab6b3295eb7a45161 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/A/d74ea2d1c4bb4dcab6b3295eb7a45161 2024-11-25T05:55:20,768 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/33947c0ba9824521b3f65897e8de63d5 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/33947c0ba9824521b3f65897e8de63d5 2024-11-25T05:55:20,769 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/354047ed3b984dd68228ba90a204aee3 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/354047ed3b984dd68228ba90a204aee3 2024-11-25T05:55:20,770 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/4b076b4611ce46bd8b644663c90971bc to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/4b076b4611ce46bd8b644663c90971bc 2024-11-25T05:55:20,771 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/dfe8f92814c74607bcaa5e0a34d11ed8 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/B/dfe8f92814c74607bcaa5e0a34d11ed8 2024-11-25T05:55:20,773 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/2867882cf4d945e294be1c4a1a86bdfb to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/2867882cf4d945e294be1c4a1a86bdfb 2024-11-25T05:55:20,774 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/4e3a6645201f4eb2b8a75b24e5c9ff86 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/4e3a6645201f4eb2b8a75b24e5c9ff86 2024-11-25T05:55:20,775 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/8b06ce67b9a24d3ca0b9ad78a48d5bba to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/8b06ce67b9a24d3ca0b9ad78a48d5bba 2024-11-25T05:55:20,776 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/b2138007688d40039cabcd6a29f74ff9 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/C/b2138007688d40039cabcd6a29f74ff9 2024-11-25T05:55:20,778 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/recovered.edits/753.seqid to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34/recovered.edits/753.seqid 2024-11-25T05:55:20,779 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/cf8ce3f4f3ad5ac4b5297b0ca903cd34 2024-11-25T05:55:20,779 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-25T05:55:20,784 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T05:55:20,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-25T05:55:20,793 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-25T05:55:20,827 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-25T05:55:20,830 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T05:55:20,831 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-25T05:55:20,831 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732514120831"}]},"ts":"9223372036854775807"} 2024-11-25T05:55:20,836 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-25T05:55:20,836 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => cf8ce3f4f3ad5ac4b5297b0ca903cd34, NAME => 'TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T05:55:20,836 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-25T05:55:20,837 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732514120836"}]},"ts":"9223372036854775807"} 2024-11-25T05:55:20,840 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-25T05:55:20,842 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T05:55:20,844 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 96 msec 2024-11-25T05:55:20,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-25T05:55:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-25T05:55:20,863 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:55:20,881 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=247 (was 210) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1220560945_22 at /127.0.0.1:54428 [Waiting for operation #438] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;8ef925b832e3:35599-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1646263908_22 at /127.0.0.1:54278 [Waiting for operation #451] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=351 (was 88) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7060 (was 7722) 2024-11-25T05:55:20,892 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=351, ProcessCount=11, AvailableMemoryMB=7058 2024-11-25T05:55:20,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T05:55:20,894 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T05:55:20,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-25T05:55:20,896 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T05:55:20,896 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:55:20,897 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-25T05:55:20,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-25T05:55:20,897 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T05:55:20,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741980_1156 (size=1061) 2024-11-25T05:55:21,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-25T05:55:21,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-25T05:55:21,305 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a 2024-11-25T05:55:21,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741981_1157 (size=53) 2024-11-25T05:55:21,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-25T05:55:21,711 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T05:55:21,711 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f2c8004e9247eeca508bdec4ef4bbd6d, disabling compactions & flushes 2024-11-25T05:55:21,711 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:21,711 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:21,711 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. after waiting 0 ms 2024-11-25T05:55:21,711 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:21,711 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:21,711 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f2c8004e9247eeca508bdec4ef4bbd6d: Waiting for close lock at 1732514121711Disabling compacts and flushes for region at 1732514121711Disabling writes for close at 1732514121711Writing region close event to WAL at 1732514121711Closed at 1732514121711 2024-11-25T05:55:21,713 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T05:55:21,713 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732514121713"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732514121713"}]},"ts":"1732514121713"} 2024-11-25T05:55:21,715 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T05:55:21,716 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T05:55:21,717 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732514121716"}]},"ts":"1732514121716"} 2024-11-25T05:55:21,718 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-25T05:55:21,719 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f2c8004e9247eeca508bdec4ef4bbd6d, ASSIGN}] 2024-11-25T05:55:21,720 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f2c8004e9247eeca508bdec4ef4bbd6d, ASSIGN 2024-11-25T05:55:21,720 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f2c8004e9247eeca508bdec4ef4bbd6d, ASSIGN; state=OFFLINE, location=8ef925b832e3,35599,1732514089005; forceNewPlan=false, retain=false 2024-11-25T05:55:21,871 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=f2c8004e9247eeca508bdec4ef4bbd6d, regionState=OPENING, regionLocation=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:21,875 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f2c8004e9247eeca508bdec4ef4bbd6d, ASSIGN because future has completed 2024-11-25T05:55:21,877 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005}] 2024-11-25T05:55:22,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-25T05:55:22,036 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:22,036 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} 2024-11-25T05:55:22,036 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:22,036 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T05:55:22,037 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:22,037 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:22,039 INFO [StoreOpener-f2c8004e9247eeca508bdec4ef4bbd6d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:22,040 INFO [StoreOpener-f2c8004e9247eeca508bdec4ef4bbd6d-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T05:55:22,041 INFO [StoreOpener-f2c8004e9247eeca508bdec4ef4bbd6d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2c8004e9247eeca508bdec4ef4bbd6d columnFamilyName A 2024-11-25T05:55:22,042 DEBUG [StoreOpener-f2c8004e9247eeca508bdec4ef4bbd6d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:55:22,043 INFO [StoreOpener-f2c8004e9247eeca508bdec4ef4bbd6d-1 {}] regionserver.HStore(327): Store=f2c8004e9247eeca508bdec4ef4bbd6d/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T05:55:22,043 INFO [StoreOpener-f2c8004e9247eeca508bdec4ef4bbd6d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:22,044 INFO [StoreOpener-f2c8004e9247eeca508bdec4ef4bbd6d-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T05:55:22,045 INFO [StoreOpener-f2c8004e9247eeca508bdec4ef4bbd6d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2c8004e9247eeca508bdec4ef4bbd6d columnFamilyName B 2024-11-25T05:55:22,045 DEBUG [StoreOpener-f2c8004e9247eeca508bdec4ef4bbd6d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:55:22,045 INFO [StoreOpener-f2c8004e9247eeca508bdec4ef4bbd6d-1 {}] regionserver.HStore(327): Store=f2c8004e9247eeca508bdec4ef4bbd6d/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T05:55:22,045 INFO [StoreOpener-f2c8004e9247eeca508bdec4ef4bbd6d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:22,046 INFO [StoreOpener-f2c8004e9247eeca508bdec4ef4bbd6d-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T05:55:22,046 INFO [StoreOpener-f2c8004e9247eeca508bdec4ef4bbd6d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2c8004e9247eeca508bdec4ef4bbd6d columnFamilyName C 2024-11-25T05:55:22,046 DEBUG [StoreOpener-f2c8004e9247eeca508bdec4ef4bbd6d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:55:22,047 INFO [StoreOpener-f2c8004e9247eeca508bdec4ef4bbd6d-1 {}] regionserver.HStore(327): Store=f2c8004e9247eeca508bdec4ef4bbd6d/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T05:55:22,047 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:22,047 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:22,048 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:22,048 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:22,049 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:22,049 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:22,050 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T05:55:22,051 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:22,052 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T05:55:22,053 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened f2c8004e9247eeca508bdec4ef4bbd6d; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71251948, jitterRate=0.06173676252365112}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T05:55:22,053 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:22,054 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for f2c8004e9247eeca508bdec4ef4bbd6d: Running coprocessor pre-open hook at 1732514122037Writing region info on filesystem at 1732514122037Initializing all the Stores at 1732514122038 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732514122038Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732514122038Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732514122038Cleaning up temporary data from old regions at 1732514122049 (+11 ms)Running coprocessor post-open hooks at 1732514122053 (+4 ms)Region opened successfully at 1732514122054 (+1 ms) 2024-11-25T05:55:22,054 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., pid=32, masterSystemTime=1732514122031 2024-11-25T05:55:22,056 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:22,056 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:22,057 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=f2c8004e9247eeca508bdec4ef4bbd6d, regionState=OPEN, openSeqNum=2, regionLocation=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:22,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 because future has completed 2024-11-25T05:55:22,061 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-25T05:55:22,061 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 in 183 msec 2024-11-25T05:55:22,062 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-25T05:55:22,062 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f2c8004e9247eeca508bdec4ef4bbd6d, ASSIGN in 342 msec 2024-11-25T05:55:22,063 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T05:55:22,063 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732514122063"}]},"ts":"1732514122063"} 2024-11-25T05:55:22,065 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-25T05:55:22,066 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T05:55:22,067 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1720 sec 2024-11-25T05:55:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-25T05:55:23,042 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:55:23,043 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@328db2e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:23,043 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:55:23,043 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:55:23,044 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:55:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:55:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:55:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63097a23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:55:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:55:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:23,046 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59174, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:55:23,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63bdc2aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:23,047 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:55:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:55:23,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:55:23,050 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39646, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:55:23,052 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T05:55:23,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 8ef925b832e3,41285,1732514088402 2024-11-25T05:55:23,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@47569338 2024-11-25T05:55:23,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T05:55:23,054 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59186, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T05:55:23,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T05:55:23,059 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T05:55:23,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-25T05:55:23,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741982_1158 (size=1097) 2024-11-25T05:55:23,478 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-25T05:55:23,478 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-25T05:55:23,482 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T05:55:23,488 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f2c8004e9247eeca508bdec4ef4bbd6d, REOPEN/MOVE}] 2024-11-25T05:55:23,488 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f2c8004e9247eeca508bdec4ef4bbd6d, REOPEN/MOVE 2024-11-25T05:55:23,489 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=f2c8004e9247eeca508bdec4ef4bbd6d, regionState=CLOSING, regionLocation=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:23,491 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f2c8004e9247eeca508bdec4ef4bbd6d, REOPEN/MOVE because future has completed 2024-11-25T05:55:23,491 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T05:55:23,492 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005}] 2024-11-25T05:55:23,644 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:23,644 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T05:55:23,644 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing f2c8004e9247eeca508bdec4ef4bbd6d, disabling compactions & flushes 2024-11-25T05:55:23,644 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:23,644 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:23,644 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. after waiting 0 ms 2024-11-25T05:55:23,644 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:23,648 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T05:55:23,649 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:23,649 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for f2c8004e9247eeca508bdec4ef4bbd6d: Waiting for close lock at 1732514123644Running coprocessor pre-close hooks at 1732514123644Disabling compacts and flushes for region at 1732514123644Disabling writes for close at 1732514123644Writing region close event to WAL at 1732514123645 (+1 ms)Running coprocessor post-close hooks at 1732514123649 (+4 ms)Closed at 1732514123649 2024-11-25T05:55:23,649 WARN [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: f2c8004e9247eeca508bdec4ef4bbd6d to self. 2024-11-25T05:55:23,651 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:23,651 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=f2c8004e9247eeca508bdec4ef4bbd6d, regionState=CLOSED 2024-11-25T05:55:23,652 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 because future has completed 2024-11-25T05:55:23,655 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-25T05:55:23,655 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 in 162 msec 2024-11-25T05:55:23,656 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f2c8004e9247eeca508bdec4ef4bbd6d, REOPEN/MOVE; state=CLOSED, location=8ef925b832e3,35599,1732514089005; forceNewPlan=false, retain=true 2024-11-25T05:55:23,807 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=f2c8004e9247eeca508bdec4ef4bbd6d, regionState=OPENING, regionLocation=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:23,810 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f2c8004e9247eeca508bdec4ef4bbd6d, REOPEN/MOVE because future has completed 2024-11-25T05:55:23,811 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005}] 2024-11-25T05:55:23,983 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:23,983 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} 2024-11-25T05:55:23,984 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:23,984 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T05:55:23,984 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:23,984 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:23,986 INFO [StoreOpener-f2c8004e9247eeca508bdec4ef4bbd6d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:23,987 INFO [StoreOpener-f2c8004e9247eeca508bdec4ef4bbd6d-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T05:55:23,992 INFO [StoreOpener-f2c8004e9247eeca508bdec4ef4bbd6d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2c8004e9247eeca508bdec4ef4bbd6d columnFamilyName A 2024-11-25T05:55:23,994 DEBUG [StoreOpener-f2c8004e9247eeca508bdec4ef4bbd6d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:55:23,995 INFO [StoreOpener-f2c8004e9247eeca508bdec4ef4bbd6d-1 {}] regionserver.HStore(327): Store=f2c8004e9247eeca508bdec4ef4bbd6d/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T05:55:23,995 INFO [StoreOpener-f2c8004e9247eeca508bdec4ef4bbd6d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:23,996 INFO [StoreOpener-f2c8004e9247eeca508bdec4ef4bbd6d-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T05:55:23,996 INFO [StoreOpener-f2c8004e9247eeca508bdec4ef4bbd6d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2c8004e9247eeca508bdec4ef4bbd6d columnFamilyName B 2024-11-25T05:55:23,996 DEBUG [StoreOpener-f2c8004e9247eeca508bdec4ef4bbd6d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:55:23,997 INFO [StoreOpener-f2c8004e9247eeca508bdec4ef4bbd6d-1 {}] regionserver.HStore(327): Store=f2c8004e9247eeca508bdec4ef4bbd6d/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T05:55:23,997 INFO [StoreOpener-f2c8004e9247eeca508bdec4ef4bbd6d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:23,998 INFO [StoreOpener-f2c8004e9247eeca508bdec4ef4bbd6d-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T05:55:23,998 INFO [StoreOpener-f2c8004e9247eeca508bdec4ef4bbd6d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2c8004e9247eeca508bdec4ef4bbd6d columnFamilyName C 2024-11-25T05:55:23,998 DEBUG [StoreOpener-f2c8004e9247eeca508bdec4ef4bbd6d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:55:23,998 INFO [StoreOpener-f2c8004e9247eeca508bdec4ef4bbd6d-1 {}] regionserver.HStore(327): Store=f2c8004e9247eeca508bdec4ef4bbd6d/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T05:55:23,999 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:23,999 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:23,999 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:24,000 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:24,001 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:24,001 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:24,002 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T05:55:24,003 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:24,004 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened f2c8004e9247eeca508bdec4ef4bbd6d; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67284452, jitterRate=0.0026164650917053223}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T05:55:24,004 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:24,005 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for f2c8004e9247eeca508bdec4ef4bbd6d: Running coprocessor pre-open hook at 1732514123984Writing region info on filesystem at 1732514123984Initializing all the Stores at 1732514123985 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732514123985Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732514123985Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732514123985Cleaning up temporary data from old regions at 1732514124001 (+16 ms)Running coprocessor post-open hooks at 1732514124004 (+3 ms)Region opened successfully at 1732514124005 (+1 ms) 2024-11-25T05:55:24,005 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., pid=37, masterSystemTime=1732514123975 2024-11-25T05:55:24,008 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:24,008 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:24,008 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=f2c8004e9247eeca508bdec4ef4bbd6d, regionState=OPEN, openSeqNum=5, regionLocation=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:24,010 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 because future has completed 2024-11-25T05:55:24,014 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-25T05:55:24,014 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 in 201 msec 2024-11-25T05:55:24,016 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-25T05:55:24,016 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f2c8004e9247eeca508bdec4ef4bbd6d, REOPEN/MOVE in 527 msec 2024-11-25T05:55:24,020 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-25T05:55:24,020 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 536 msec 2024-11-25T05:55:24,025 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 960 msec 2024-11-25T05:55:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-25T05:55:24,027 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:55:24,028 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@263f8ca5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:24,028 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:55:24,028 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:55:24,029 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:55:24,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:55:24,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:55:24,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cf0b33b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:24,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:55:24,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:55:24,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:24,032 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59204, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:55:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3146dbb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:24,033 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65b7b5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:24,033 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:55:24,033 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:55:24,034 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:55:24,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:55:24,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:55:24,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a87beba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:24,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:55:24,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:55:24,036 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:24,037 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:55:24,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7525736d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:24,038 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a1845b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:24,039 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:55:24,039 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:55:24,040 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:55:24,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:55:24,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:55:24,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@314070cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:24,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:55:24,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:55:24,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:24,041 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59234, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:55:24,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40086eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:24,043 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b0484de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:24,044 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:55:24,044 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:55:24,045 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:55:24,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:55:24,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:55:24,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d1afb77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:24,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:55:24,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:55:24,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:24,047 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59252, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:55:24,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@308e68da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:24,049 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@349085e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:24,049 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:55:24,049 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:55:24,050 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:55:24,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:55:24,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:55:24,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ab06ba8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:24,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:55:24,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:55:24,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:24,052 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59276, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:55:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d34e723, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:24,057 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26dcfede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:24,057 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:55:24,057 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:55:24,058 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:55:24,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:55:24,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:55:24,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d7569d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:24,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:55:24,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:55:24,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:24,060 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59300, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:55:24,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c010b97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:24,062 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d21c225, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:24,062 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:55:24,062 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:55:24,063 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:55:24,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:55:24,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:55:24,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64877518, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:24,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:55:24,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:55:24,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:24,065 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:55:24,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40682de9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:24,066 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a92e196, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:24,066 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:55:24,066 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:55:24,068 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:55:24,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:55:24,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:55:24,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ae94d25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:24,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:55:24,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:55:24,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:24,070 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59340, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:55:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50c6f230, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:24,071 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3806f988, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:24,071 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:55:24,071 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:55:24,072 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:55:24,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:55:24,073 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:55:24,073 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e1cfdae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:24,073 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:55:24,073 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:55:24,073 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:24,074 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59358, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:55:24,075 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ba1929, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:24,076 DEBUG [Thread-764 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:55:24,076 DEBUG [Thread-768 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:55:24,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:55:24,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:55:24,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:55:24,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:55:24,078 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39652, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:55:24,080 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39664, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:55:24,081 DEBUG [Thread-762 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:55:24,082 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:55:24,083 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:55:24,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5] 2024-11-25T05:55:24,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5] 2024-11-25T05:55:24,084 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39668, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:55:24,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5] 2024-11-25T05:55:24,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:24,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f2c8004e9247eeca508bdec4ef4bbd6d 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-25T05:55:24,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=A 2024-11-25T05:55:24,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:24,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=B 2024-11-25T05:55:24,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:24,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=C 2024-11-25T05:55:24,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:24,100 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:55:24,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-25T05:55:24,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-25T05:55:24,103 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:55:24,104 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:55:24,104 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:55:24,115 DEBUG [Thread-777 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:55:24,115 DEBUG [Thread-779 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:55:24,116 DEBUG [Thread-777 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:55:24,116 DEBUG [Thread-779 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:55:24,116 DEBUG [Thread-779 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:55:24,116 DEBUG [Thread-777 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:55:24,117 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39672, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:55:24,117 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39670, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:55:24,118 DEBUG [Thread-775 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:55:24,119 DEBUG [Thread-775 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:55:24,120 DEBUG [Thread-775 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:55:24,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5] 2024-11-25T05:55:24,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732514124107, max timestamp=1732514124113Num uniques 859; 2024-11-25T05:55:24,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T05:55:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5] 2024-11-25T05:55:24,122 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39692, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:55:24,123 DEBUG [Thread-766 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:55:24,123 DEBUG [Thread-770 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:55:24,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:55:24,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:55:24,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5] 2024-11-25T05:55:24,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:55:24,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:55:24,125 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:55:24,126 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39712, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:55:24,126 DEBUG [Thread-773 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:55:24,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732514124107, max timestamp=1732514124113Num uniques 859; 2024-11-25T05:55:24,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T05:55:24,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732514124107, max timestamp=1732514124113Num uniques 859; 2024-11-25T05:55:24,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T05:55:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:55:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5] 2024-11-25T05:55:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:55:24,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5] 2024-11-25T05:55:24,129 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39716, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:55:24,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5] 2024-11-25T05:55:24,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125cfb145670883406eb836e14aed4598a3_f2c8004e9247eeca508bdec4ef4bbd6d is 50, key is test_row_0/A:col10/1732514124090/Put/seqid=0 2024-11-25T05:55:24,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-25T05:55:24,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:24,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514184197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:24,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:24,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:24,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514184213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:24,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:24,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514184215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:24,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:24,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:24,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:24,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514184217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:24,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:24,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:24,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514184218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:24,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:24,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741983_1159 (size=12154) 2024-11-25T05:55:24,242 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:55:24,249 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125cfb145670883406eb836e14aed4598a3_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125cfb145670883406eb836e14aed4598a3_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:24,251 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/5a01127926b0442a85cbae79e2514164, store: [table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:24,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T05:55:24,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:24,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:24,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:24,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:24,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:24,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/5a01127926b0442a85cbae79e2514164 is 175, key is test_row_0/A:col10/1732514124090/Put/seqid=0 2024-11-25T05:55:24,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:24,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741984_1160 (size=30955) 2024-11-25T05:55:24,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:24,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514184342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:24,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:24,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:24,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514184342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:24,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:24,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:24,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514184343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:24,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:24,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:24,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514184343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:24,371 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,371 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,371 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:24,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:24,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514184322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:24,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:24,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T05:55:24,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:24,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:24,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:24,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:24,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:24,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:24,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-25T05:55:24,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T05:55:24,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:24,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:24,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:24,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:24,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:24,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:24,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514184572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:24,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514184572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:24,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:24,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:24,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514184583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514184583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:24,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514184589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:24,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:24,707 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/5a01127926b0442a85cbae79e2514164 2024-11-25T05:55:24,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T05:55:24,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:24,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:24,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:24,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:24,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:24,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-25T05:55:24,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/9b8e964ffa6441869b2318349ed663e8 is 50, key is test_row_0/B:col10/1732514124090/Put/seqid=0 2024-11-25T05:55:24,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741985_1161 (size=12001) 2024-11-25T05:55:24,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T05:55:24,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:24,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:24,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:24,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:24,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:24,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:24,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514184892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:24,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:24,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:24,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514184892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:24,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:24,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:24,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514184893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:24,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:24,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:24,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514184893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:24,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:24,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514184902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:24,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:24,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:25,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T05:55:25,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:25,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:25,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:25,028 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:25,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:25,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:25,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T05:55:25,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:25,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:25,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:25,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:25,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:25,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:25,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/9b8e964ffa6441869b2318349ed663e8 2024-11-25T05:55:25,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-25T05:55:25,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/a4131ce91f3d453ab62e0fbc0ea20517 is 50, key is test_row_0/C:col10/1732514124090/Put/seqid=0 2024-11-25T05:55:25,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741986_1162 (size=12001) 2024-11-25T05:55:25,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/a4131ce91f3d453ab62e0fbc0ea20517 2024-11-25T05:55:25,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/5a01127926b0442a85cbae79e2514164 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/5a01127926b0442a85cbae79e2514164 2024-11-25T05:55:25,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T05:55:25,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:25,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:25,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:25,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:25,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:25,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:25,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/5a01127926b0442a85cbae79e2514164, entries=150, sequenceid=15, filesize=30.2 K 2024-11-25T05:55:25,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/9b8e964ffa6441869b2318349ed663e8 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/9b8e964ffa6441869b2318349ed663e8 2024-11-25T05:55:25,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/9b8e964ffa6441869b2318349ed663e8, entries=150, sequenceid=15, filesize=11.7 K 2024-11-25T05:55:25,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/a4131ce91f3d453ab62e0fbc0ea20517 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/a4131ce91f3d453ab62e0fbc0ea20517 2024-11-25T05:55:25,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/a4131ce91f3d453ab62e0fbc0ea20517, entries=150, sequenceid=15, filesize=11.7 K 2024-11-25T05:55:25,401 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for f2c8004e9247eeca508bdec4ef4bbd6d in 1309ms, sequenceid=15, compaction requested=false 2024-11-25T05:55:25,402 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-25T05:55:25,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:25,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:25,424 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f2c8004e9247eeca508bdec4ef4bbd6d 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-25T05:55:25,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732514124202, max timestamp=1732514125423Num uniques 1068; 2024-11-25T05:55:25,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-25T05:55:25,435 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732514124202, max timestamp=1732514125423Num uniques 1068; 2024-11-25T05:55:25,435 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-25T05:55:25,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=A 2024-11-25T05:55:25,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:55:25,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=B 2024-11-25T05:55:25,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:55:25,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=C 2024-11-25T05:55:25,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-25T05:55:25,474 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T05:55:25,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411258144ae1a13434e99b6c774e043759289_f2c8004e9247eeca508bdec4ef4bbd6d is 50, key is test_row_0/A:col10/1732514124202/Put/seqid=0 2024-11-25T05:55:25,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T05:55:25,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:25,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:25,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:25,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:25,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:25,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:25,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741987_1163 (size=19474) 2024-11-25T05:55:25,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:25,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514185554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:25,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:25,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:25,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:25,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:25,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:25,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514185559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:25,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514185558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:25,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:25,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:25,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:25,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:25,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514185561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:25,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:25,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:25,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:25,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:25,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514185563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:25,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T05:55:25,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:25,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:25,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:25,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:25,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:25,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:25,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:25,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514185672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:25,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:25,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:25,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:25,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:25,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514185673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:25,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:25,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:25,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:25,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:25,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514185673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:25,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:25,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:25,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:25,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:25,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514185673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:25,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:25,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:25,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:25,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:25,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514185673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:25,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:25,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:25,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:25,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T05:55:25,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:25,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:25,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:25,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:25,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:25,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:25,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:25,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514185882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:25,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:25,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:25,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:25,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514185882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:25,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:25,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:25,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:25,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514185882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:25,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:25,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:25,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:25,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:25,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:25,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:25,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514185892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:25,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:25,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:25,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:25,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:25,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514185895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:25,901 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:25,901 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:25,901 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:25,930 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:55:25,941 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411258144ae1a13434e99b6c774e043759289_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411258144ae1a13434e99b6c774e043759289_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:25,943 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/685cc15eae92481d98fa917dd88a78dd, store: [table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:25,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/685cc15eae92481d98fa917dd88a78dd is 175, key is test_row_0/A:col10/1732514124202/Put/seqid=0 2024-11-25T05:55:25,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T05:55:25,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:25,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:25,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:25,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:25,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:25,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:25,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741988_1164 (size=56733) 2024-11-25T05:55:26,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T05:55:26,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:26,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:26,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:26,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:26,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:26,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:26,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:26,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514186192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:26,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:26,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:26,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:26,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:26,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514186192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:26,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:26,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:26,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:26,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:26,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514186193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:26,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:26,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:26,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:26,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:26,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514186202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:26,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:26,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:26,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:26,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514186212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:26,218 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:26,218 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:26,218 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-25T05:55:26,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T05:55:26,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:26,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:26,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:26,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:26,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:26,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:26,392 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/685cc15eae92481d98fa917dd88a78dd 2024-11-25T05:55:26,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/d13afc3dadbe4f5392154c36cd2e516f is 50, key is test_row_0/B:col10/1732514124202/Put/seqid=0 2024-11-25T05:55:26,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T05:55:26,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:26,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:26,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:26,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:26,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:26,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741989_1165 (size=12001) 2024-11-25T05:55:26,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T05:55:26,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:26,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:26,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:26,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:26,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:26,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514186712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:26,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:26,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514186712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:26,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:26,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:26,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:26,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:26,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:26,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514186713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:26,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:26,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:26,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514186713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:26,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:26,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:26,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:26,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:26,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T05:55:26,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:26,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:26,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:26,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:26,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:26,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:26,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:26,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514186732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:26,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:26,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:26,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:26,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/d13afc3dadbe4f5392154c36cd2e516f 2024-11-25T05:55:26,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/765522b74ae74ab2acfd813aebe94b5a is 50, key is test_row_0/C:col10/1732514124202/Put/seqid=0 2024-11-25T05:55:26,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T05:55:26,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:26,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:26,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:26,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:26,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:26,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:26,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741990_1166 (size=12001) 2024-11-25T05:55:27,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T05:55:27,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:27,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:27,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:27,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:27,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:27,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T05:55:27,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:27,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:27,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:27,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:27,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:27,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/765522b74ae74ab2acfd813aebe94b5a 2024-11-25T05:55:27,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/685cc15eae92481d98fa917dd88a78dd as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/685cc15eae92481d98fa917dd88a78dd 2024-11-25T05:55:27,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/685cc15eae92481d98fa917dd88a78dd, entries=300, sequenceid=53, filesize=55.4 K 2024-11-25T05:55:27,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/d13afc3dadbe4f5392154c36cd2e516f as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/d13afc3dadbe4f5392154c36cd2e516f 2024-11-25T05:55:27,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/d13afc3dadbe4f5392154c36cd2e516f, entries=150, sequenceid=53, filesize=11.7 K 2024-11-25T05:55:27,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/765522b74ae74ab2acfd813aebe94b5a as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/765522b74ae74ab2acfd813aebe94b5a 2024-11-25T05:55:27,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/765522b74ae74ab2acfd813aebe94b5a, entries=150, sequenceid=53, filesize=11.7 K 2024-11-25T05:55:27,333 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~76.04 KB/77860, heapSize ~204.14 KB/209040, currentSize=114.05 KB/116790 for f2c8004e9247eeca508bdec4ef4bbd6d in 1910ms, sequenceid=53, compaction requested=false 2024-11-25T05:55:27,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:27,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T05:55:27,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:27,349 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing f2c8004e9247eeca508bdec4ef4bbd6d 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-25T05:55:27,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=A 2024-11-25T05:55:27,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:27,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=B 2024-11-25T05:55:27,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:27,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=C 2024-11-25T05:55:27,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:27,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411251f752419b31343138cd848e92d2e4648_f2c8004e9247eeca508bdec4ef4bbd6d is 50, key is test_row_0/A:col10/1732514125514/Put/seqid=0 2024-11-25T05:55:27,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741991_1167 (size=12154) 2024-11-25T05:55:27,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:55:27,384 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411251f752419b31343138cd848e92d2e4648_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411251f752419b31343138cd848e92d2e4648_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:27,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/80f2ee2c7b934f81b4acd7670d3768f2, store: [table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:27,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/80f2ee2c7b934f81b4acd7670d3768f2 is 175, key is test_row_0/A:col10/1732514125514/Put/seqid=0 2024-11-25T05:55:27,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741992_1168 (size=30955) 2024-11-25T05:55:27,398 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=73, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/80f2ee2c7b934f81b4acd7670d3768f2 2024-11-25T05:55:27,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/ea57d71e91c44a1f98dbc3a5007a7231 is 50, key is test_row_0/B:col10/1732514125514/Put/seqid=0 2024-11-25T05:55:27,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741993_1169 (size=12001) 2024-11-25T05:55:27,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:27,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:27,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:27,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514187739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:27,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:27,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:27,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:27,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:27,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514187740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:27,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:27,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:27,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:27,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:27,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514187740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:27,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:27,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:27,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:27,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:27,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514187743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:27,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:27,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:27,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:27,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:27,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514187753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:27,758 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_2' in f2c8004e9247eeca508bdec4ef4bbd6d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:55:27,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:27,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:27,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:27,820 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/ea57d71e91c44a1f98dbc3a5007a7231 2024-11-25T05:55:27,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/450328120f824282b10c3213bebf3adc is 50, key is test_row_0/C:col10/1732514125514/Put/seqid=0 2024-11-25T05:55:27,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:27,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514187852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:27,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:27,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:27,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:27,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741994_1170 (size=12001) 2024-11-25T05:55:27,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:27,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514187863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:27,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:27,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514187863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:27,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:27,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514187863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:27,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:27,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:27,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:27,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:27,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:27,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:27,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:27,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:27,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:28,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:28,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514188062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:28,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:28,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:28,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:28,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:28,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514188071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:28,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:28,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:28,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:28,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:28,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514188072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:28,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:28,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:28,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514188072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:28,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:28,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:28,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:28,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:28,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:28,256 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/450328120f824282b10c3213bebf3adc 2024-11-25T05:55:28,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-25T05:55:28,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/80f2ee2c7b934f81b4acd7670d3768f2 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/80f2ee2c7b934f81b4acd7670d3768f2 2024-11-25T05:55:28,277 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/80f2ee2c7b934f81b4acd7670d3768f2, entries=150, sequenceid=73, filesize=30.2 K 2024-11-25T05:55:28,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/ea57d71e91c44a1f98dbc3a5007a7231 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/ea57d71e91c44a1f98dbc3a5007a7231 2024-11-25T05:55:28,284 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/ea57d71e91c44a1f98dbc3a5007a7231, entries=150, sequenceid=73, filesize=11.7 K 2024-11-25T05:55:28,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/450328120f824282b10c3213bebf3adc as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/450328120f824282b10c3213bebf3adc 2024-11-25T05:55:28,293 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/450328120f824282b10c3213bebf3adc, entries=150, sequenceid=73, filesize=11.7 K 2024-11-25T05:55:28,294 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for f2c8004e9247eeca508bdec4ef4bbd6d in 946ms, sequenceid=73, compaction requested=true 2024-11-25T05:55:28,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:28,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:28,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-25T05:55:28,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-25T05:55:28,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-25T05:55:28,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.1930 sec 2024-11-25T05:55:28,301 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 4.2000 sec 2024-11-25T05:55:28,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:28,375 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f2c8004e9247eeca508bdec4ef4bbd6d 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-25T05:55:28,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=A 2024-11-25T05:55:28,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:28,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=B 2024-11-25T05:55:28,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:28,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=C 2024-11-25T05:55:28,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:28,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112557dbf9726dae4ed5a34ad6f2ad5d6570_f2c8004e9247eeca508bdec4ef4bbd6d is 50, key is test_row_0/A:col10/1732514127739/Put/seqid=0 2024-11-25T05:55:28,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:28,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514188408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:28,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:28,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514188409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:28,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:28,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514188414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:28,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:28,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514188418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:28,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741995_1171 (size=14594) 2024-11-25T05:55:28,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514188522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:28,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:28,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514188523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:28,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:28,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:28,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:28,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:28,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:28,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:28,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514188532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:28,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:28,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:28,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:28,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:28,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514188532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:28,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:28,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:28,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:28,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:28,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514188732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:28,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:28,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:28,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:28,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:28,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514188732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:28,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:28,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:28,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:28,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514188742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:28,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514188742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:28,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:28,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:28,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:28,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:28,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:28,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:28,852 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:55:28,861 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112557dbf9726dae4ed5a34ad6f2ad5d6570_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112557dbf9726dae4ed5a34ad6f2ad5d6570_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:28,864 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/9b6b8449514d4d59833fcd9bbc629ff6, store: [table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:28,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/9b6b8449514d4d59833fcd9bbc629ff6 is 175, key is test_row_0/A:col10/1732514127739/Put/seqid=0 2024-11-25T05:55:28,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741996_1172 (size=39549) 2024-11-25T05:55:28,873 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=86, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/9b6b8449514d4d59833fcd9bbc629ff6 2024-11-25T05:55:28,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/db31d4e994e84addbb4625ede1d6442b is 50, key is test_row_0/B:col10/1732514127739/Put/seqid=0 2024-11-25T05:55:28,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741997_1173 (size=12001) 2024-11-25T05:55:28,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/db31d4e994e84addbb4625ede1d6442b 2024-11-25T05:55:28,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/567a5473f5834acfb31d882b0c61e568 is 50, key is test_row_0/C:col10/1732514127739/Put/seqid=0 2024-11-25T05:55:28,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741998_1174 (size=12001) 2024-11-25T05:55:29,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:29,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514189052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:29,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:29,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:29,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:29,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:29,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514189052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:29,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:29,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:29,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:29,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514189052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:29,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514189053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:29,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:29,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:29,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:29,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:29,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:29,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:29,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/567a5473f5834acfb31d882b0c61e568 2024-11-25T05:55:29,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/9b6b8449514d4d59833fcd9bbc629ff6 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/9b6b8449514d4d59833fcd9bbc629ff6 2024-11-25T05:55:29,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/9b6b8449514d4d59833fcd9bbc629ff6, entries=200, sequenceid=86, filesize=38.6 K 2024-11-25T05:55:29,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/db31d4e994e84addbb4625ede1d6442b as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/db31d4e994e84addbb4625ede1d6442b 2024-11-25T05:55:29,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/db31d4e994e84addbb4625ede1d6442b, entries=150, sequenceid=86, filesize=11.7 K 2024-11-25T05:55:29,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/567a5473f5834acfb31d882b0c61e568 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/567a5473f5834acfb31d882b0c61e568 2024-11-25T05:55:29,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/567a5473f5834acfb31d882b0c61e568, entries=150, sequenceid=86, filesize=11.7 K 2024-11-25T05:55:29,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f2c8004e9247eeca508bdec4ef4bbd6d in 990ms, sequenceid=86, compaction requested=true 2024-11-25T05:55:29,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:29,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f2c8004e9247eeca508bdec4ef4bbd6d:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:55:29,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:29,366 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T05:55:29,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f2c8004e9247eeca508bdec4ef4bbd6d:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:55:29,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:29,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f2c8004e9247eeca508bdec4ef4bbd6d:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:55:29,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T05:55:29,366 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T05:55:29,367 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 158192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T05:55:29,367 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): f2c8004e9247eeca508bdec4ef4bbd6d/A is initiating minor compaction (all files) 2024-11-25T05:55:29,367 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f2c8004e9247eeca508bdec4ef4bbd6d/A in TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:29,368 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/5a01127926b0442a85cbae79e2514164, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/685cc15eae92481d98fa917dd88a78dd, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/80f2ee2c7b934f81b4acd7670d3768f2, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/9b6b8449514d4d59833fcd9bbc629ff6] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp, totalSize=154.5 K 2024-11-25T05:55:29,368 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:29,368 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. files: [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/5a01127926b0442a85cbae79e2514164, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/685cc15eae92481d98fa917dd88a78dd, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/80f2ee2c7b934f81b4acd7670d3768f2, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/9b6b8449514d4d59833fcd9bbc629ff6] 2024-11-25T05:55:29,368 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T05:55:29,368 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): f2c8004e9247eeca508bdec4ef4bbd6d/B is initiating minor compaction (all files) 2024-11-25T05:55:29,368 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f2c8004e9247eeca508bdec4ef4bbd6d/B in TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:29,369 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/9b8e964ffa6441869b2318349ed663e8, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/d13afc3dadbe4f5392154c36cd2e516f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/ea57d71e91c44a1f98dbc3a5007a7231, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/db31d4e994e84addbb4625ede1d6442b] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp, totalSize=46.9 K 2024-11-25T05:55:29,369 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a01127926b0442a85cbae79e2514164, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732514124087 2024-11-25T05:55:29,369 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b8e964ffa6441869b2318349ed663e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732514124087 2024-11-25T05:55:29,369 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 685cc15eae92481d98fa917dd88a78dd, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732514124202 2024-11-25T05:55:29,370 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting d13afc3dadbe4f5392154c36cd2e516f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732514124202 2024-11-25T05:55:29,370 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80f2ee2c7b934f81b4acd7670d3768f2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732514125514 2024-11-25T05:55:29,370 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting ea57d71e91c44a1f98dbc3a5007a7231, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732514125514 2024-11-25T05:55:29,370 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b6b8449514d4d59833fcd9bbc629ff6, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1732514127727 2024-11-25T05:55:29,370 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting db31d4e994e84addbb4625ede1d6442b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1732514127737 2024-11-25T05:55:29,399 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:29,404 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f2c8004e9247eeca508bdec4ef4bbd6d#B#compaction#156 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:29,404 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/693a19c514ae487da2bd1ad541fdb7a8 is 50, key is test_row_0/B:col10/1732514127739/Put/seqid=0 2024-11-25T05:55:29,409 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411251a3bfb8f92304580aea7f56a19deae69_f2c8004e9247eeca508bdec4ef4bbd6d store=[table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:29,415 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411251a3bfb8f92304580aea7f56a19deae69_f2c8004e9247eeca508bdec4ef4bbd6d, store=[table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:29,415 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411251a3bfb8f92304580aea7f56a19deae69_f2c8004e9247eeca508bdec4ef4bbd6d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:29,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741999_1175 (size=12139) 2024-11-25T05:55:29,457 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/693a19c514ae487da2bd1ad541fdb7a8 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/693a19c514ae487da2bd1ad541fdb7a8 2024-11-25T05:55:29,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742000_1176 (size=4469) 2024-11-25T05:55:29,465 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f2c8004e9247eeca508bdec4ef4bbd6d/B of f2c8004e9247eeca508bdec4ef4bbd6d into 693a19c514ae487da2bd1ad541fdb7a8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:29,465 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:29,465 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., storeName=f2c8004e9247eeca508bdec4ef4bbd6d/B, priority=12, startTime=1732514129366; duration=0sec 2024-11-25T05:55:29,465 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:29,465 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f2c8004e9247eeca508bdec4ef4bbd6d:B 2024-11-25T05:55:29,465 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T05:55:29,467 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f2c8004e9247eeca508bdec4ef4bbd6d#A#compaction#155 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:29,467 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/0efcbc2e796643dfa5de09370bb55f1b is 175, key is test_row_0/A:col10/1732514127739/Put/seqid=0 2024-11-25T05:55:29,468 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T05:55:29,468 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): f2c8004e9247eeca508bdec4ef4bbd6d/C is initiating minor compaction (all files) 2024-11-25T05:55:29,468 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f2c8004e9247eeca508bdec4ef4bbd6d/C in TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:29,468 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/a4131ce91f3d453ab62e0fbc0ea20517, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/765522b74ae74ab2acfd813aebe94b5a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/450328120f824282b10c3213bebf3adc, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/567a5473f5834acfb31d882b0c61e568] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp, totalSize=46.9 K 2024-11-25T05:55:29,469 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting a4131ce91f3d453ab62e0fbc0ea20517, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732514124087 2024-11-25T05:55:29,470 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 765522b74ae74ab2acfd813aebe94b5a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732514124202 2024-11-25T05:55:29,470 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 450328120f824282b10c3213bebf3adc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732514125514 2024-11-25T05:55:29,471 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 567a5473f5834acfb31d882b0c61e568, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1732514127737 2024-11-25T05:55:29,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742001_1177 (size=31093) 2024-11-25T05:55:29,503 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f2c8004e9247eeca508bdec4ef4bbd6d#C#compaction#157 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:29,504 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/0c3c8f9a08744d1da1d6d5526146d0dd is 50, key is test_row_0/C:col10/1732514127739/Put/seqid=0 2024-11-25T05:55:29,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742002_1178 (size=12139) 2024-11-25T05:55:29,546 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/0c3c8f9a08744d1da1d6d5526146d0dd as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/0c3c8f9a08744d1da1d6d5526146d0dd 2024-11-25T05:55:29,553 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f2c8004e9247eeca508bdec4ef4bbd6d/C of f2c8004e9247eeca508bdec4ef4bbd6d into 0c3c8f9a08744d1da1d6d5526146d0dd(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:29,553 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:29,553 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., storeName=f2c8004e9247eeca508bdec4ef4bbd6d/C, priority=12, startTime=1732514129366; duration=0sec 2024-11-25T05:55:29,553 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:29,553 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f2c8004e9247eeca508bdec4ef4bbd6d:C 2024-11-25T05:55:29,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:29,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f2c8004e9247eeca508bdec4ef4bbd6d 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-25T05:55:29,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=A 2024-11-25T05:55:29,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:29,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=B 2024-11-25T05:55:29,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:29,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=C 2024-11-25T05:55:29,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:29,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125bc78ed8ddd074bd68c1c7e224bea176c_f2c8004e9247eeca508bdec4ef4bbd6d is 50, key is test_row_0/A:col10/1732514129569/Put/seqid=0 2024-11-25T05:55:29,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:29,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514189581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:29,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:29,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:29,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:29,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:29,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514189582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:29,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:29,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514189583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:29,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:29,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:29,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:29,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:29,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:29,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:29,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514189592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:29,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:29,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:29,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:29,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742003_1179 (size=12154) 2024-11-25T05:55:29,609 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:55:29,615 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125bc78ed8ddd074bd68c1c7e224bea176c_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125bc78ed8ddd074bd68c1c7e224bea176c_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:29,619 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/05a05c8f7eb9471e8f6c8a53026be233, store: [table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:29,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/05a05c8f7eb9471e8f6c8a53026be233 is 175, key is test_row_0/A:col10/1732514129569/Put/seqid=0 2024-11-25T05:55:29,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742004_1180 (size=30955) 2024-11-25T05:55:29,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:29,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514189702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:29,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:29,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:29,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:29,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:29,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514189702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:29,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:29,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514189702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:29,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:29,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514189702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:29,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:29,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:29,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:29,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:29,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514189782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:29,786 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_2' in f2c8004e9247eeca508bdec4ef4bbd6d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:55:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:29,901 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/0efcbc2e796643dfa5de09370bb55f1b as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/0efcbc2e796643dfa5de09370bb55f1b 2024-11-25T05:55:29,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:29,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514189920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:29,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:29,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514189920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:29,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:29,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514189920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:29,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:29,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:29,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:29,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:29,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:29,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:29,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:29,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:29,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:29,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:29,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514189921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:29,922 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f2c8004e9247eeca508bdec4ef4bbd6d/A of f2c8004e9247eeca508bdec4ef4bbd6d into 0efcbc2e796643dfa5de09370bb55f1b(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:29,922 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:29,922 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., storeName=f2c8004e9247eeca508bdec4ef4bbd6d/A, priority=12, startTime=1732514129365; duration=0sec 2024-11-25T05:55:29,922 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:29,922 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f2c8004e9247eeca508bdec4ef4bbd6d:A 2024-11-25T05:55:29,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:29,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:29,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:30,043 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=108, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/05a05c8f7eb9471e8f6c8a53026be233 2024-11-25T05:55:30,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/520a28ec3efd4749bffbcd40e1a22728 is 50, key is test_row_0/B:col10/1732514129569/Put/seqid=0 2024-11-25T05:55:30,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742005_1181 (size=12001) 2024-11-25T05:55:30,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:30,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514190232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:30,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:30,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:30,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:30,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:30,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514190233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:30,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:30,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:30,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514190233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:30,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:30,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:30,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:30,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:30,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514190234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:30,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:30,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:30,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:30,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/520a28ec3efd4749bffbcd40e1a22728 2024-11-25T05:55:30,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/dca88a81fb8e474892a5160274f56a55 is 50, key is test_row_0/C:col10/1732514129569/Put/seqid=0 2024-11-25T05:55:30,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742006_1182 (size=12001) 2024-11-25T05:55:30,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/dca88a81fb8e474892a5160274f56a55 2024-11-25T05:55:30,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/05a05c8f7eb9471e8f6c8a53026be233 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/05a05c8f7eb9471e8f6c8a53026be233 2024-11-25T05:55:30,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/05a05c8f7eb9471e8f6c8a53026be233, entries=150, sequenceid=108, filesize=30.2 K 2024-11-25T05:55:30,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/520a28ec3efd4749bffbcd40e1a22728 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/520a28ec3efd4749bffbcd40e1a22728 2024-11-25T05:55:30,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/520a28ec3efd4749bffbcd40e1a22728, entries=150, sequenceid=108, filesize=11.7 K 2024-11-25T05:55:30,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/dca88a81fb8e474892a5160274f56a55 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/dca88a81fb8e474892a5160274f56a55 2024-11-25T05:55:30,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/dca88a81fb8e474892a5160274f56a55, entries=150, sequenceid=108, filesize=11.7 K 2024-11-25T05:55:30,528 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for f2c8004e9247eeca508bdec4ef4bbd6d in 956ms, sequenceid=108, compaction requested=false 2024-11-25T05:55:30,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:30,749 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f2c8004e9247eeca508bdec4ef4bbd6d 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-25T05:55:30,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:30,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=A 2024-11-25T05:55:30,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:30,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=B 2024-11-25T05:55:30,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:30,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=C 2024-11-25T05:55:30,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:30,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125de1e67c45fe844b7b5d3146807a83127_f2c8004e9247eeca508bdec4ef4bbd6d is 50, key is test_row_0/A:col10/1732514129589/Put/seqid=0 2024-11-25T05:55:30,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742007_1183 (size=14594) 2024-11-25T05:55:30,805 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:55:30,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:30,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:30,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514190805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:30,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514190811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:30,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:30,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:30,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:30,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:30,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:30,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:30,825 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125de1e67c45fe844b7b5d3146807a83127_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125de1e67c45fe844b7b5d3146807a83127_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:30,827 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/a35856942c794b30adecbd7c41ae30df, store: [table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:30,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/a35856942c794b30adecbd7c41ae30df is 175, key is test_row_0/A:col10/1732514129589/Put/seqid=0 2024-11-25T05:55:30,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:30,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514190818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:30,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:30,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514190819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:30,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:30,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:30,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:30,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:30,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742008_1184 (size=39549) 2024-11-25T05:55:30,867 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=123, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/a35856942c794b30adecbd7c41ae30df 2024-11-25T05:55:30,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/86f2b20e7d7046d1b19ca9583b3a8e6f is 50, key is test_row_0/B:col10/1732514129589/Put/seqid=0 2024-11-25T05:55:30,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:30,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514190922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:30,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:30,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:30,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:30,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514190922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:30,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:30,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742009_1185 (size=12001) 2024-11-25T05:55:30,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:30,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:30,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:30,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/86f2b20e7d7046d1b19ca9583b3a8e6f 2024-11-25T05:55:30,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/ece4f0d968604d17b1068419ec5d3ff7 is 50, key is test_row_0/C:col10/1732514129589/Put/seqid=0 2024-11-25T05:55:30,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742010_1186 (size=12001) 2024-11-25T05:55:30,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:30,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514190942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:30,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:30,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:30,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:30,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514190942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:30,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:30,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:30,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:30,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:30,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/ece4f0d968604d17b1068419ec5d3ff7 2024-11-25T05:55:30,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/a35856942c794b30adecbd7c41ae30df as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/a35856942c794b30adecbd7c41ae30df 2024-11-25T05:55:30,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/a35856942c794b30adecbd7c41ae30df, entries=200, sequenceid=123, filesize=38.6 K 2024-11-25T05:55:30,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/86f2b20e7d7046d1b19ca9583b3a8e6f as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/86f2b20e7d7046d1b19ca9583b3a8e6f 2024-11-25T05:55:30,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/86f2b20e7d7046d1b19ca9583b3a8e6f, entries=150, sequenceid=123, filesize=11.7 K 2024-11-25T05:55:30,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/ece4f0d968604d17b1068419ec5d3ff7 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/ece4f0d968604d17b1068419ec5d3ff7 2024-11-25T05:55:30,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/ece4f0d968604d17b1068419ec5d3ff7, entries=150, sequenceid=123, filesize=11.7 K 2024-11-25T05:55:30,983 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for f2c8004e9247eeca508bdec4ef4bbd6d in 234ms, sequenceid=123, compaction requested=true 2024-11-25T05:55:30,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:30,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f2c8004e9247eeca508bdec4ef4bbd6d:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:55:30,983 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:30,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:30,983 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:30,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f2c8004e9247eeca508bdec4ef4bbd6d:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:55:30,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:30,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f2c8004e9247eeca508bdec4ef4bbd6d:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:55:30,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:30,985 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101597 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:30,985 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:30,985 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): f2c8004e9247eeca508bdec4ef4bbd6d/A is initiating minor compaction (all files) 2024-11-25T05:55:30,985 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): f2c8004e9247eeca508bdec4ef4bbd6d/B is initiating minor compaction (all files) 2024-11-25T05:55:30,985 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f2c8004e9247eeca508bdec4ef4bbd6d/B in TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:30,985 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f2c8004e9247eeca508bdec4ef4bbd6d/A in TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:30,985 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/693a19c514ae487da2bd1ad541fdb7a8, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/520a28ec3efd4749bffbcd40e1a22728, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/86f2b20e7d7046d1b19ca9583b3a8e6f] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp, totalSize=35.3 K 2024-11-25T05:55:30,985 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/0efcbc2e796643dfa5de09370bb55f1b, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/05a05c8f7eb9471e8f6c8a53026be233, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/a35856942c794b30adecbd7c41ae30df] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp, totalSize=99.2 K 2024-11-25T05:55:30,986 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:30,986 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. files: [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/0efcbc2e796643dfa5de09370bb55f1b, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/05a05c8f7eb9471e8f6c8a53026be233, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/a35856942c794b30adecbd7c41ae30df] 2024-11-25T05:55:30,986 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 693a19c514ae487da2bd1ad541fdb7a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1732514127737 2024-11-25T05:55:30,986 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0efcbc2e796643dfa5de09370bb55f1b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1732514127737 2024-11-25T05:55:30,986 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05a05c8f7eb9471e8f6c8a53026be233, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732514128411 2024-11-25T05:55:30,986 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 520a28ec3efd4749bffbcd40e1a22728, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732514128411 2024-11-25T05:55:30,987 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting a35856942c794b30adecbd7c41ae30df, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732514129580 2024-11-25T05:55:30,987 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 86f2b20e7d7046d1b19ca9583b3a8e6f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732514129589 2024-11-25T05:55:30,997 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:31,002 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411256f634777821242d28af5f160a93b5b42_f2c8004e9247eeca508bdec4ef4bbd6d store=[table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:31,003 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f2c8004e9247eeca508bdec4ef4bbd6d#B#compaction#165 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:31,004 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/95ce0136df784c03a3bb5d1b9b9216b9 is 50, key is test_row_0/B:col10/1732514129589/Put/seqid=0 2024-11-25T05:55:31,011 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411256f634777821242d28af5f160a93b5b42_f2c8004e9247eeca508bdec4ef4bbd6d, store=[table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:31,012 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411256f634777821242d28af5f160a93b5b42_f2c8004e9247eeca508bdec4ef4bbd6d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:31,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742011_1187 (size=12241) 2024-11-25T05:55:31,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742012_1188 (size=4469) 2024-11-25T05:55:31,030 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f2c8004e9247eeca508bdec4ef4bbd6d#A#compaction#164 average throughput is 0.74 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:31,037 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/b442f0fc0d334651864a29b7d17eba55 is 175, key is test_row_0/A:col10/1732514129589/Put/seqid=0 2024-11-25T05:55:31,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742013_1189 (size=31195) 2024-11-25T05:55:31,053 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/b442f0fc0d334651864a29b7d17eba55 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/b442f0fc0d334651864a29b7d17eba55 2024-11-25T05:55:31,063 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f2c8004e9247eeca508bdec4ef4bbd6d/A of f2c8004e9247eeca508bdec4ef4bbd6d into b442f0fc0d334651864a29b7d17eba55(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:31,064 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:31,064 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., storeName=f2c8004e9247eeca508bdec4ef4bbd6d/A, priority=13, startTime=1732514130983; duration=0sec 2024-11-25T05:55:31,064 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:31,064 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f2c8004e9247eeca508bdec4ef4bbd6d:A 2024-11-25T05:55:31,064 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:31,067 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:31,067 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): f2c8004e9247eeca508bdec4ef4bbd6d/C is initiating minor compaction (all files) 2024-11-25T05:55:31,067 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f2c8004e9247eeca508bdec4ef4bbd6d/C in TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:31,067 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/0c3c8f9a08744d1da1d6d5526146d0dd, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/dca88a81fb8e474892a5160274f56a55, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/ece4f0d968604d17b1068419ec5d3ff7] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp, totalSize=35.3 K 2024-11-25T05:55:31,068 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c3c8f9a08744d1da1d6d5526146d0dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1732514127737 2024-11-25T05:55:31,068 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting dca88a81fb8e474892a5160274f56a55, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732514128411 2024-11-25T05:55:31,068 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting ece4f0d968604d17b1068419ec5d3ff7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732514129589 2024-11-25T05:55:31,078 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f2c8004e9247eeca508bdec4ef4bbd6d#C#compaction#166 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:31,084 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/2a781b8b83684f659229820d17b1539e is 50, key is test_row_0/C:col10/1732514129589/Put/seqid=0 2024-11-25T05:55:31,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742014_1190 (size=12241) 2024-11-25T05:55:31,129 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/2a781b8b83684f659229820d17b1539e as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/2a781b8b83684f659229820d17b1539e 2024-11-25T05:55:31,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:31,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f2c8004e9247eeca508bdec4ef4bbd6d 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-25T05:55:31,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=A 2024-11-25T05:55:31,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:31,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=B 2024-11-25T05:55:31,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:31,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=C 2024-11-25T05:55:31,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:31,140 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f2c8004e9247eeca508bdec4ef4bbd6d/C of f2c8004e9247eeca508bdec4ef4bbd6d into 2a781b8b83684f659229820d17b1539e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:31,140 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:31,141 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., storeName=f2c8004e9247eeca508bdec4ef4bbd6d/C, priority=13, startTime=1732514130984; duration=0sec 2024-11-25T05:55:31,141 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:31,141 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f2c8004e9247eeca508bdec4ef4bbd6d:C 2024-11-25T05:55:31,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125dba0c80ce5484e158cb5be88d18aeae2_f2c8004e9247eeca508bdec4ef4bbd6d is 50, key is test_row_0/A:col10/1732514130814/Put/seqid=0 2024-11-25T05:55:31,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:31,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514191162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:31,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:31,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:31,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:31,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:31,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514191171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:31,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:31,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:31,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:31,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:31,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514191171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:31,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:31,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514191172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:31,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:31,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:31,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:31,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:31,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:31,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:31,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742015_1191 (size=14794) 2024-11-25T05:55:31,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:31,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514191282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:31,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:31,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514191282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:31,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:31,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:31,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:31,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:31,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:31,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:31,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:31,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514191282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:31,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:31,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:31,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:31,430 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/95ce0136df784c03a3bb5d1b9b9216b9 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/95ce0136df784c03a3bb5d1b9b9216b9 2024-11-25T05:55:31,439 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f2c8004e9247eeca508bdec4ef4bbd6d/B of f2c8004e9247eeca508bdec4ef4bbd6d into 95ce0136df784c03a3bb5d1b9b9216b9(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:31,439 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:31,439 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., storeName=f2c8004e9247eeca508bdec4ef4bbd6d/B, priority=13, startTime=1732514130983; duration=0sec 2024-11-25T05:55:31,439 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:31,439 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f2c8004e9247eeca508bdec4ef4bbd6d:B 2024-11-25T05:55:31,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:31,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514191482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:31,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:31,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:31,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:31,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514191492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:31,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:31,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514191492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:31,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:31,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:31,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:31,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:31,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:31,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:31,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514191493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:31,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:31,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:31,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:31,592 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:55:31,598 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125dba0c80ce5484e158cb5be88d18aeae2_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125dba0c80ce5484e158cb5be88d18aeae2_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:31,604 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/9e7fd382dbf349a5b4f96d9ea7331b09, store: [table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:31,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/9e7fd382dbf349a5b4f96d9ea7331b09 is 175, key is test_row_0/A:col10/1732514130814/Put/seqid=0 2024-11-25T05:55:31,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742016_1192 (size=39749) 2024-11-25T05:55:31,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:31,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514191804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:31,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:31,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514191804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:31,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:31,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:31,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514191804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:31,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:31,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:31,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:31,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:31,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:31,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:31,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:31,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:31,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:31,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514191992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:31,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:31,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:31,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:32,036 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=144, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/9e7fd382dbf349a5b4f96d9ea7331b09 2024-11-25T05:55:32,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/0df351a772954dfaac2f43e95d56d55e is 50, key is test_row_0/B:col10/1732514130814/Put/seqid=0 2024-11-25T05:55:32,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742017_1193 (size=12151) 2024-11-25T05:55:32,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-25T05:55:32,283 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:55:32,283 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:55:32,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-25T05:55:32,285 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:55:32,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-25T05:55:32,286 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:55:32,286 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:55:32,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:32,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514192312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:32,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:32,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:32,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:32,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:32,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514192312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:32,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:32,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:32,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:32,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:32,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514192312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:32,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:32,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:32,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:32,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-25T05:55:32,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T05:55:32,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:32,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:32,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:32,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:32,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:32,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:32,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/0df351a772954dfaac2f43e95d56d55e 2024-11-25T05:55:32,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/c9919f146577427e9a747a405b034b15 is 50, key is test_row_0/C:col10/1732514130814/Put/seqid=0 2024-11-25T05:55:32,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742018_1194 (size=12151) 2024-11-25T05:55:32,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/c9919f146577427e9a747a405b034b15 2024-11-25T05:55:32,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/9e7fd382dbf349a5b4f96d9ea7331b09 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/9e7fd382dbf349a5b4f96d9ea7331b09 2024-11-25T05:55:32,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/9e7fd382dbf349a5b4f96d9ea7331b09, entries=200, sequenceid=144, filesize=38.8 K 2024-11-25T05:55:32,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/0df351a772954dfaac2f43e95d56d55e as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/0df351a772954dfaac2f43e95d56d55e 2024-11-25T05:55:32,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/0df351a772954dfaac2f43e95d56d55e, entries=150, sequenceid=144, filesize=11.9 K 2024-11-25T05:55:32,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/c9919f146577427e9a747a405b034b15 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/c9919f146577427e9a747a405b034b15 2024-11-25T05:55:32,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/c9919f146577427e9a747a405b034b15, entries=150, sequenceid=144, filesize=11.9 K 2024-11-25T05:55:32,504 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for f2c8004e9247eeca508bdec4ef4bbd6d in 1369ms, sequenceid=144, compaction requested=false 2024-11-25T05:55:32,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:32,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T05:55:32,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:32,592 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing f2c8004e9247eeca508bdec4ef4bbd6d 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-25T05:55:32,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=A 2024-11-25T05:55:32,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:32,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=B 2024-11-25T05:55:32,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:32,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=C 2024-11-25T05:55:32,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:32,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-25T05:55:32,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125751e35a47a044dccaeb461814adf6e20_f2c8004e9247eeca508bdec4ef4bbd6d is 50, key is test_row_0/A:col10/1732514131167/Put/seqid=0 2024-11-25T05:55:32,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742019_1195 (size=12304) 2024-11-25T05:55:32,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:55:32,619 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125751e35a47a044dccaeb461814adf6e20_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125751e35a47a044dccaeb461814adf6e20_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:32,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/d09fe96e3da14af18905d78d8567414a, store: [table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:32,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/d09fe96e3da14af18905d78d8567414a is 175, key is test_row_0/A:col10/1732514131167/Put/seqid=0 2024-11-25T05:55:32,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742020_1196 (size=31105) 2024-11-25T05:55:32,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-25T05:55:33,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:33,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:33,027 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/d09fe96e3da14af18905d78d8567414a 2024-11-25T05:55:33,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/952af1dcda4e48bd91c00454d7876723 is 50, key is test_row_0/B:col10/1732514131167/Put/seqid=0 2024-11-25T05:55:33,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742021_1197 (size=12151) 2024-11-25T05:55:33,044 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/952af1dcda4e48bd91c00454d7876723 2024-11-25T05:55:33,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/ef1f7128407548b5ad5cfd4b46c2f20a is 50, key is test_row_0/C:col10/1732514131167/Put/seqid=0 2024-11-25T05:55:33,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742022_1198 (size=12151) 2024-11-25T05:55:33,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:33,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514193076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:33,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:33,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:33,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:33,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:33,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514193193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:33,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:33,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:33,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:33,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:33,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514193322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:33,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:33,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514193323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:33,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:33,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514193323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:33,328 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_2' in f2c8004e9247eeca508bdec4ef4bbd6d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:55:33,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:33,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:33,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:33,329 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_2' in f2c8004e9247eeca508bdec4ef4bbd6d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:55:33,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:33,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:33,330 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_1' in f2c8004e9247eeca508bdec4ef4bbd6d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:55:33,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:33,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:33,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:33,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:33,374 INFO [master/8ef925b832e3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-25T05:55:33,374 INFO [master/8ef925b832e3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-25T05:55:33,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:33,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514193402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:33,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:33,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:33,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:33,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-25T05:55:33,460 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/ef1f7128407548b5ad5cfd4b46c2f20a 2024-11-25T05:55:33,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/d09fe96e3da14af18905d78d8567414a as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/d09fe96e3da14af18905d78d8567414a 2024-11-25T05:55:33,469 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/d09fe96e3da14af18905d78d8567414a, entries=150, sequenceid=158, filesize=30.4 K 2024-11-25T05:55:33,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/952af1dcda4e48bd91c00454d7876723 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/952af1dcda4e48bd91c00454d7876723 2024-11-25T05:55:33,478 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/952af1dcda4e48bd91c00454d7876723, entries=150, sequenceid=158, filesize=11.9 K 2024-11-25T05:55:33,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/ef1f7128407548b5ad5cfd4b46c2f20a as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/ef1f7128407548b5ad5cfd4b46c2f20a 2024-11-25T05:55:33,486 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/ef1f7128407548b5ad5cfd4b46c2f20a, entries=150, sequenceid=158, filesize=11.9 K 2024-11-25T05:55:33,488 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f2c8004e9247eeca508bdec4ef4bbd6d in 895ms, sequenceid=158, compaction requested=true 2024-11-25T05:55:33,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:33,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:33,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-25T05:55:33,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-25T05:55:33,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-25T05:55:33,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2030 sec 2024-11-25T05:55:33,493 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.2080 sec 2024-11-25T05:55:33,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732514133047, max timestamp=1732514133712Num uniques 859; 2024-11-25T05:55:33,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T05:55:33,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732514133047, max timestamp=1732514133712Num uniques 859; 2024-11-25T05:55:33,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T05:55:33,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732514133047, max timestamp=1732514133712Num uniques 859; 2024-11-25T05:55:33,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T05:55:33,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:33,731 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f2c8004e9247eeca508bdec4ef4bbd6d 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-11-25T05:55:33,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=A 2024-11-25T05:55:33,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:55:33,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=B 2024-11-25T05:55:33,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:55:33,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=C 2024-11-25T05:55:33,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:55:33,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125e63fdd6efb0e4634af54a8d0bdc29833_f2c8004e9247eeca508bdec4ef4bbd6d is 50, key is test_row_0/A:col10/1732514133728/Put/seqid=0 2024-11-25T05:55:33,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742023_1199 (size=14794) 2024-11-25T05:55:33,748 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:55:33,767 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125e63fdd6efb0e4634af54a8d0bdc29833_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125e63fdd6efb0e4634af54a8d0bdc29833_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:33,768 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/0e023da5c29b46eda0c3391fa6e9709f, store: [table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:33,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/0e023da5c29b46eda0c3391fa6e9709f is 175, key is test_row_0/A:col10/1732514133728/Put/seqid=0 2024-11-25T05:55:33,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742024_1200 (size=39749) 2024-11-25T05:55:33,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732514133800, max timestamp=1732514133826Num uniques 859; 2024-11-25T05:55:33,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732514133800, max timestamp=1732514133826Num uniques 859; 2024-11-25T05:55:33,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732514133800, max timestamp=1732514133826Num uniques 859; 2024-11-25T05:55:33,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-25T05:55:33,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-25T05:55:33,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-25T05:55:33,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:33,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514193877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:33,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:33,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:33,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:33,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:33,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514193883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:33,889 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:33,889 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:33,889 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:33,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:33,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514193992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:33,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:33,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:33,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:33,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:33,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514193992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:33,998 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:33,998 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:33,998 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:34,176 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=183, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/0e023da5c29b46eda0c3391fa6e9709f 2024-11-25T05:55:34,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/343745c1087c4d86941ef5f977630ceb is 50, key is test_row_0/B:col10/1732514133728/Put/seqid=0 2024-11-25T05:55:34,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742025_1201 (size=12151) 2024-11-25T05:55:34,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:34,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514194202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:34,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/343745c1087c4d86941ef5f977630ceb 2024-11-25T05:55:34,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:34,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:34,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:34,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:34,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514194202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:34,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:34,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:34,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:34,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/62390d50f32545bf87ad53b59a3e8e55 is 50, key is test_row_0/C:col10/1732514133728/Put/seqid=0 2024-11-25T05:55:34,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742026_1202 (size=12151) 2024-11-25T05:55:34,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-25T05:55:34,432 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:55:34,433 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:55:34,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-25T05:55:34,436 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:55:34,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-25T05:55:34,439 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:55:34,439 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:55:34,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:34,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514194512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:34,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:34,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514194512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:34,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-25T05:55:34,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-25T05:55:34,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:34,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:34,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:34,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:34,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:34,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:34,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/62390d50f32545bf87ad53b59a3e8e55 2024-11-25T05:55:34,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/0e023da5c29b46eda0c3391fa6e9709f as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/0e023da5c29b46eda0c3391fa6e9709f 2024-11-25T05:55:34,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/0e023da5c29b46eda0c3391fa6e9709f, entries=200, sequenceid=183, filesize=38.8 K 2024-11-25T05:55:34,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/343745c1087c4d86941ef5f977630ceb as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/343745c1087c4d86941ef5f977630ceb 2024-11-25T05:55:34,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/343745c1087c4d86941ef5f977630ceb, entries=150, sequenceid=183, filesize=11.9 K 2024-11-25T05:55:34,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/62390d50f32545bf87ad53b59a3e8e55 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/62390d50f32545bf87ad53b59a3e8e55 2024-11-25T05:55:34,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/62390d50f32545bf87ad53b59a3e8e55, entries=150, sequenceid=183, filesize=11.9 K 2024-11-25T05:55:34,655 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=134.18 KB/137400 for f2c8004e9247eeca508bdec4ef4bbd6d in 924ms, sequenceid=183, compaction requested=true 2024-11-25T05:55:34,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:34,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f2c8004e9247eeca508bdec4ef4bbd6d:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:55:34,656 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T05:55:34,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:34,656 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T05:55:34,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f2c8004e9247eeca508bdec4ef4bbd6d:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:55:34,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:34,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f2c8004e9247eeca508bdec4ef4bbd6d:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:55:34,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:34,658 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T05:55:34,658 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): f2c8004e9247eeca508bdec4ef4bbd6d/B is initiating minor compaction (all files) 2024-11-25T05:55:34,659 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f2c8004e9247eeca508bdec4ef4bbd6d/B in TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:34,659 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/95ce0136df784c03a3bb5d1b9b9216b9, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/0df351a772954dfaac2f43e95d56d55e, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/952af1dcda4e48bd91c00454d7876723, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/343745c1087c4d86941ef5f977630ceb] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp, totalSize=47.6 K 2024-11-25T05:55:34,659 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141798 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T05:55:34,659 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): f2c8004e9247eeca508bdec4ef4bbd6d/A is initiating minor compaction (all files) 2024-11-25T05:55:34,659 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f2c8004e9247eeca508bdec4ef4bbd6d/A in TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:34,659 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/b442f0fc0d334651864a29b7d17eba55, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/9e7fd382dbf349a5b4f96d9ea7331b09, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/d09fe96e3da14af18905d78d8567414a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/0e023da5c29b46eda0c3391fa6e9709f] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp, totalSize=138.5 K 2024-11-25T05:55:34,659 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:34,660 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. files: [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/b442f0fc0d334651864a29b7d17eba55, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/9e7fd382dbf349a5b4f96d9ea7331b09, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/d09fe96e3da14af18905d78d8567414a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/0e023da5c29b46eda0c3391fa6e9709f] 2024-11-25T05:55:34,660 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting b442f0fc0d334651864a29b7d17eba55, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732514129589 2024-11-25T05:55:34,660 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 95ce0136df784c03a3bb5d1b9b9216b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732514129589 2024-11-25T05:55:34,661 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e7fd382dbf349a5b4f96d9ea7331b09, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732514130801 2024-11-25T05:55:34,661 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 0df351a772954dfaac2f43e95d56d55e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732514130813 2024-11-25T05:55:34,661 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 952af1dcda4e48bd91c00454d7876723, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732514131155 2024-11-25T05:55:34,662 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting d09fe96e3da14af18905d78d8567414a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732514131155 2024-11-25T05:55:34,662 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 343745c1087c4d86941ef5f977630ceb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732514133720 2024-11-25T05:55:34,662 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e023da5c29b46eda0c3391fa6e9709f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732514133072 2024-11-25T05:55:34,692 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f2c8004e9247eeca508bdec4ef4bbd6d#B#compaction#176 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:34,693 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/70e317a771724227ab883a240303a0ea is 50, key is test_row_0/B:col10/1732514133728/Put/seqid=0 2024-11-25T05:55:34,702 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:34,722 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125dd5113429f2f4f08bdb8dd6c7d6c00f7_f2c8004e9247eeca508bdec4ef4bbd6d store=[table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:34,725 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125dd5113429f2f4f08bdb8dd6c7d6c00f7_f2c8004e9247eeca508bdec4ef4bbd6d, store=[table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:34,726 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125dd5113429f2f4f08bdb8dd6c7d6c00f7_f2c8004e9247eeca508bdec4ef4bbd6d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:34,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-25T05:55:34,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:34,746 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing f2c8004e9247eeca508bdec4ef4bbd6d 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-25T05:55:34,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=A 2024-11-25T05:55:34,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:55:34,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=B 2024-11-25T05:55:34,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:55:34,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=C 2024-11-25T05:55:34,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:55:34,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742027_1203 (size=12527) 2024-11-25T05:55:34,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-25T05:55:34,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742028_1204 (size=4469) 2024-11-25T05:55:34,789 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f2c8004e9247eeca508bdec4ef4bbd6d#A#compaction#177 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:34,790 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/7a8cf08465854f128704000f05be0a09 is 175, key is test_row_0/A:col10/1732514133728/Put/seqid=0 2024-11-25T05:55:34,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411254dd3aa7a831a4396821d9bd8000495dd_f2c8004e9247eeca508bdec4ef4bbd6d is 50, key is test_row_0/A:col10/1732514133882/Put/seqid=0 2024-11-25T05:55:34,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742029_1205 (size=31481) 2024-11-25T05:55:34,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742030_1206 (size=12304) 2024-11-25T05:55:34,827 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/7a8cf08465854f128704000f05be0a09 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/7a8cf08465854f128704000f05be0a09 2024-11-25T05:55:34,832 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f2c8004e9247eeca508bdec4ef4bbd6d/A of f2c8004e9247eeca508bdec4ef4bbd6d into 7a8cf08465854f128704000f05be0a09(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:34,832 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:34,832 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., storeName=f2c8004e9247eeca508bdec4ef4bbd6d/A, priority=12, startTime=1732514134655; duration=0sec 2024-11-25T05:55:34,833 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:34,833 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f2c8004e9247eeca508bdec4ef4bbd6d:A 2024-11-25T05:55:34,833 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T05:55:34,836 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T05:55:34,836 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): f2c8004e9247eeca508bdec4ef4bbd6d/C is initiating minor compaction (all files) 2024-11-25T05:55:34,836 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f2c8004e9247eeca508bdec4ef4bbd6d/C in TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:34,836 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/2a781b8b83684f659229820d17b1539e, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/c9919f146577427e9a747a405b034b15, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/ef1f7128407548b5ad5cfd4b46c2f20a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/62390d50f32545bf87ad53b59a3e8e55] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp, totalSize=47.6 K 2024-11-25T05:55:34,836 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a781b8b83684f659229820d17b1539e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732514129589 2024-11-25T05:55:34,837 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9919f146577427e9a747a405b034b15, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732514130813 2024-11-25T05:55:34,838 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef1f7128407548b5ad5cfd4b46c2f20a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732514131155 2024-11-25T05:55:34,838 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62390d50f32545bf87ad53b59a3e8e55, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732514133720 2024-11-25T05:55:34,868 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f2c8004e9247eeca508bdec4ef4bbd6d#C#compaction#179 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:34,869 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/3aaa18bbf8004b8db496df90670f4f5b is 50, key is test_row_0/C:col10/1732514133728/Put/seqid=0 2024-11-25T05:55:34,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742031_1207 (size=12527) 2024-11-25T05:55:35,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:35,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:35,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-25T05:55:35,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:35,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514195050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:35,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:35,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514195052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:35,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:35,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:35,159 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/70e317a771724227ab883a240303a0ea as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/70e317a771724227ab883a240303a0ea 2024-11-25T05:55:35,165 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f2c8004e9247eeca508bdec4ef4bbd6d/B of f2c8004e9247eeca508bdec4ef4bbd6d into 70e317a771724227ab883a240303a0ea(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:35,165 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:35,165 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., storeName=f2c8004e9247eeca508bdec4ef4bbd6d/B, priority=12, startTime=1732514134656; duration=0sec 2024-11-25T05:55:35,165 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:35,165 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f2c8004e9247eeca508bdec4ef4bbd6d:B 2024-11-25T05:55:35,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:35,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514195172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:35,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:35,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514195172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:35,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:35,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:35,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:55:35,226 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411254dd3aa7a831a4396821d9bd8000495dd_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411254dd3aa7a831a4396821d9bd8000495dd_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:35,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/c98d7a8acadc432aba153c5f2f7dcf18, store: [table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:35,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/c98d7a8acadc432aba153c5f2f7dcf18 is 175, key is test_row_0/A:col10/1732514133882/Put/seqid=0 2024-11-25T05:55:35,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742032_1208 (size=31105) 2024-11-25T05:55:35,234 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=218, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/c98d7a8acadc432aba153c5f2f7dcf18 2024-11-25T05:55:35,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/f86ff5697aac491cb5b841dc0a06b4ec is 50, key is test_row_0/B:col10/1732514133882/Put/seqid=0 2024-11-25T05:55:35,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742033_1209 (size=12151) 2024-11-25T05:55:35,251 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/f86ff5697aac491cb5b841dc0a06b4ec 2024-11-25T05:55:35,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/e149f426913346a396982f955692f0d7 is 50, key is test_row_0/C:col10/1732514133882/Put/seqid=0 2024-11-25T05:55:35,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742034_1210 (size=12151) 2024-11-25T05:55:35,275 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/e149f426913346a396982f955692f0d7 2024-11-25T05:55:35,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/c98d7a8acadc432aba153c5f2f7dcf18 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/c98d7a8acadc432aba153c5f2f7dcf18 2024-11-25T05:55:35,286 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/c98d7a8acadc432aba153c5f2f7dcf18, entries=150, sequenceid=218, filesize=30.4 K 2024-11-25T05:55:35,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/f86ff5697aac491cb5b841dc0a06b4ec as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/f86ff5697aac491cb5b841dc0a06b4ec 2024-11-25T05:55:35,293 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/f86ff5697aac491cb5b841dc0a06b4ec, entries=150, sequenceid=218, filesize=11.9 K 2024-11-25T05:55:35,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/e149f426913346a396982f955692f0d7 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/e149f426913346a396982f955692f0d7 2024-11-25T05:55:35,298 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/e149f426913346a396982f955692f0d7, entries=150, sequenceid=218, filesize=11.9 K 2024-11-25T05:55:35,298 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for f2c8004e9247eeca508bdec4ef4bbd6d in 552ms, sequenceid=218, compaction requested=false 2024-11-25T05:55:35,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:35,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:35,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-25T05:55:35,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-25T05:55:35,302 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-25T05:55:35,302 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 861 msec 2024-11-25T05:55:35,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 870 msec 2024-11-25T05:55:35,310 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/3aaa18bbf8004b8db496df90670f4f5b as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/3aaa18bbf8004b8db496df90670f4f5b 2024-11-25T05:55:35,316 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f2c8004e9247eeca508bdec4ef4bbd6d/C of f2c8004e9247eeca508bdec4ef4bbd6d into 3aaa18bbf8004b8db496df90670f4f5b(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:35,316 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:35,316 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., storeName=f2c8004e9247eeca508bdec4ef4bbd6d/C, priority=12, startTime=1732514134656; duration=0sec 2024-11-25T05:55:35,316 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:35,316 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f2c8004e9247eeca508bdec4ef4bbd6d:C 2024-11-25T05:55:35,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:35,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f2c8004e9247eeca508bdec4ef4bbd6d 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-25T05:55:35,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=A 2024-11-25T05:55:35,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:35,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=B 2024-11-25T05:55:35,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:35,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=C 2024-11-25T05:55:35,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:35,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411252d48b6bbc38d4c6ead19be20153e6411_f2c8004e9247eeca508bdec4ef4bbd6d is 50, key is test_row_0/A:col10/1732514135050/Put/seqid=0 2024-11-25T05:55:35,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742035_1211 (size=14794) 2024-11-25T05:55:35,380 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:55:35,386 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411252d48b6bbc38d4c6ead19be20153e6411_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252d48b6bbc38d4c6ead19be20153e6411_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:35,387 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/514ead0d11644c4d8b0e89ad3d5a33a5, store: [table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:35,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/514ead0d11644c4d8b0e89ad3d5a33a5 is 175, key is test_row_0/A:col10/1732514135050/Put/seqid=0 2024-11-25T05:55:35,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:35,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514195394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:35,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:35,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514195395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:35,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:35,403 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514195395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:35,403 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:35,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:35,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514195397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:35,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514195397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:35,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742036_1212 (size=39749) 2024-11-25T05:55:35,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:35,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:35,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514195512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:35,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:35,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514195513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:35,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:35,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:35,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514195513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:35,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:35,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:35,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:35,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514195513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:35,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:35,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514195513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:35,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:35,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-25T05:55:35,573 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:55:35,573 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:55:35,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-25T05:55:35,575 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:55:35,575 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:55:35,575 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:55:35,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-25T05:55:35,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-25T05:55:35,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514195722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:35,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:35,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514195722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:35,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:35,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514195722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:35,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:35,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-25T05:55:35,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514195722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:35,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:35,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:35,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:35,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:35,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:35,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514195722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:35,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:35,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:35,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:35,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:35,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:35,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:35,806 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/514ead0d11644c4d8b0e89ad3d5a33a5 2024-11-25T05:55:35,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/9ceeb009ef904c26a93584ebf5ef15d1 is 50, key is test_row_0/B:col10/1732514135050/Put/seqid=0 2024-11-25T05:55:35,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742037_1213 (size=12151) 2024-11-25T05:55:35,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/9ceeb009ef904c26a93584ebf5ef15d1 2024-11-25T05:55:35,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/efbe8c1f8f1c4cf5aa937bbe9e65095c is 50, key is test_row_0/C:col10/1732514135050/Put/seqid=0 2024-11-25T05:55:35,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742038_1214 (size=12151) 2024-11-25T05:55:35,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-25T05:55:35,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:35,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:35,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:35,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:35,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-25T05:55:36,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-25T05:55:36,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:36,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:36,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:36,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:36,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:36,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:36,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:36,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514196032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:36,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514196032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:36,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:36,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514196032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514196032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:36,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514196032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:36,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-25T05:55:36,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:36,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:36,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:36,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:36,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:36,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:36,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-25T05:55:36,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/efbe8c1f8f1c4cf5aa937bbe9e65095c 2024-11-25T05:55:36,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/514ead0d11644c4d8b0e89ad3d5a33a5 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/514ead0d11644c4d8b0e89ad3d5a33a5 2024-11-25T05:55:36,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/514ead0d11644c4d8b0e89ad3d5a33a5, entries=200, sequenceid=233, filesize=38.8 K 2024-11-25T05:55:36,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/9ceeb009ef904c26a93584ebf5ef15d1 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/9ceeb009ef904c26a93584ebf5ef15d1 2024-11-25T05:55:36,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/9ceeb009ef904c26a93584ebf5ef15d1, entries=150, sequenceid=233, filesize=11.9 K 2024-11-25T05:55:36,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/efbe8c1f8f1c4cf5aa937bbe9e65095c as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/efbe8c1f8f1c4cf5aa937bbe9e65095c 2024-11-25T05:55:36,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/efbe8c1f8f1c4cf5aa937bbe9e65095c, entries=150, sequenceid=233, filesize=11.9 K 2024-11-25T05:55:36,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f2c8004e9247eeca508bdec4ef4bbd6d in 930ms, sequenceid=233, compaction requested=true 2024-11-25T05:55:36,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:36,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f2c8004e9247eeca508bdec4ef4bbd6d:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:55:36,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:36,278 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:36,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f2c8004e9247eeca508bdec4ef4bbd6d:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:55:36,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:36,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f2c8004e9247eeca508bdec4ef4bbd6d:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:55:36,278 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:36,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:36,281 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:36,281 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): f2c8004e9247eeca508bdec4ef4bbd6d/B is initiating minor compaction (all files) 2024-11-25T05:55:36,281 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f2c8004e9247eeca508bdec4ef4bbd6d/B in TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:36,281 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/70e317a771724227ab883a240303a0ea, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/f86ff5697aac491cb5b841dc0a06b4ec, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/9ceeb009ef904c26a93584ebf5ef15d1] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp, totalSize=36.0 K 2024-11-25T05:55:36,281 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102335 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:36,281 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): f2c8004e9247eeca508bdec4ef4bbd6d/A is initiating minor compaction (all files) 2024-11-25T05:55:36,281 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f2c8004e9247eeca508bdec4ef4bbd6d/A in TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:36,281 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/7a8cf08465854f128704000f05be0a09, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/c98d7a8acadc432aba153c5f2f7dcf18, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/514ead0d11644c4d8b0e89ad3d5a33a5] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp, totalSize=99.9 K 2024-11-25T05:55:36,281 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 70e317a771724227ab883a240303a0ea, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732514133720 2024-11-25T05:55:36,282 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:36,282 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. files: [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/7a8cf08465854f128704000f05be0a09, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/c98d7a8acadc432aba153c5f2f7dcf18, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/514ead0d11644c4d8b0e89ad3d5a33a5] 2024-11-25T05:55:36,282 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting f86ff5697aac491cb5b841dc0a06b4ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732514133851 2024-11-25T05:55:36,282 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a8cf08465854f128704000f05be0a09, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732514133720 2024-11-25T05:55:36,282 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting c98d7a8acadc432aba153c5f2f7dcf18, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732514133851 2024-11-25T05:55:36,282 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ceeb009ef904c26a93584ebf5ef15d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732514135041 2024-11-25T05:55:36,283 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 514ead0d11644c4d8b0e89ad3d5a33a5, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732514135041 2024-11-25T05:55:36,294 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:36,299 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112539677fdbb4b94128b83a1f8ca660cdd9_f2c8004e9247eeca508bdec4ef4bbd6d store=[table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:36,302 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112539677fdbb4b94128b83a1f8ca660cdd9_f2c8004e9247eeca508bdec4ef4bbd6d, store=[table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:36,303 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112539677fdbb4b94128b83a1f8ca660cdd9_f2c8004e9247eeca508bdec4ef4bbd6d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:36,307 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f2c8004e9247eeca508bdec4ef4bbd6d#B#compaction#186 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:36,307 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/cf722fc8d349423e949f1fbd12d0feb7 is 50, key is test_row_0/B:col10/1732514135050/Put/seqid=0 2024-11-25T05:55:36,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742039_1215 (size=4469) 2024-11-25T05:55:36,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742040_1216 (size=12629) 2024-11-25T05:55:36,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-25T05:55:36,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:36,339 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing f2c8004e9247eeca508bdec4ef4bbd6d 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-25T05:55:36,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=A 2024-11-25T05:55:36,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:36,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=B 2024-11-25T05:55:36,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:36,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=C 2024-11-25T05:55:36,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:36,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411251ee6d90e831741e08914fd4b625b2af1_f2c8004e9247eeca508bdec4ef4bbd6d is 50, key is test_row_0/A:col10/1732514135395/Put/seqid=0 2024-11-25T05:55:36,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742041_1217 (size=12304) 2024-11-25T05:55:36,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:36,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:36,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:36,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514196552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:36,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:36,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:36,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514196554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:36,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:36,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:36,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514196554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:36,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:36,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:36,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514196554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:36,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:36,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:36,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514196555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:36,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:36,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514196662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:36,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:36,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514196662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:36,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:36,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:36,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:36,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514196673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:36,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:36,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514196673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:36,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:36,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:36,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514196674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:36,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:36,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:36,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-25T05:55:36,712 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f2c8004e9247eeca508bdec4ef4bbd6d#A#compaction#185 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:36,712 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/55b0e8cef01a49d7b370ce670fc4bce0 is 175, key is test_row_0/A:col10/1732514135050/Put/seqid=0 2024-11-25T05:55:36,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742042_1218 (size=31583) 2024-11-25T05:55:36,725 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/cf722fc8d349423e949f1fbd12d0feb7 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/cf722fc8d349423e949f1fbd12d0feb7 2024-11-25T05:55:36,729 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/55b0e8cef01a49d7b370ce670fc4bce0 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/55b0e8cef01a49d7b370ce670fc4bce0 2024-11-25T05:55:36,739 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f2c8004e9247eeca508bdec4ef4bbd6d/B of f2c8004e9247eeca508bdec4ef4bbd6d into cf722fc8d349423e949f1fbd12d0feb7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:36,739 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:36,739 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f2c8004e9247eeca508bdec4ef4bbd6d/A of f2c8004e9247eeca508bdec4ef4bbd6d into 55b0e8cef01a49d7b370ce670fc4bce0(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:36,739 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., storeName=f2c8004e9247eeca508bdec4ef4bbd6d/B, priority=13, startTime=1732514136278; duration=0sec 2024-11-25T05:55:36,739 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:36,739 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., storeName=f2c8004e9247eeca508bdec4ef4bbd6d/A, priority=13, startTime=1732514136278; duration=0sec 2024-11-25T05:55:36,739 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:36,739 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:36,739 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f2c8004e9247eeca508bdec4ef4bbd6d:B 2024-11-25T05:55:36,739 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f2c8004e9247eeca508bdec4ef4bbd6d:A 2024-11-25T05:55:36,739 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:36,740 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:36,740 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): f2c8004e9247eeca508bdec4ef4bbd6d/C is initiating minor compaction (all files) 2024-11-25T05:55:36,740 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f2c8004e9247eeca508bdec4ef4bbd6d/C in TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:36,741 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/3aaa18bbf8004b8db496df90670f4f5b, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/e149f426913346a396982f955692f0d7, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/efbe8c1f8f1c4cf5aa937bbe9e65095c] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp, totalSize=36.0 K 2024-11-25T05:55:36,741 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 3aaa18bbf8004b8db496df90670f4f5b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732514133720 2024-11-25T05:55:36,741 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting e149f426913346a396982f955692f0d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732514133851 2024-11-25T05:55:36,742 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting efbe8c1f8f1c4cf5aa937bbe9e65095c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732514135041 2024-11-25T05:55:36,750 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f2c8004e9247eeca508bdec4ef4bbd6d#C#compaction#188 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:36,750 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/d80602b7f2e444f3b16b75a96b89d91e is 50, key is test_row_0/C:col10/1732514135050/Put/seqid=0 2024-11-25T05:55:36,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:55:36,755 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411251ee6d90e831741e08914fd4b625b2af1_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411251ee6d90e831741e08914fd4b625b2af1_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:36,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/782bbedf211243d49ebe7bf1d645e328, store: [table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:36,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/782bbedf211243d49ebe7bf1d645e328 is 175, key is test_row_0/A:col10/1732514135395/Put/seqid=0 2024-11-25T05:55:36,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742043_1219 (size=12629) 2024-11-25T05:55:36,790 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/d80602b7f2e444f3b16b75a96b89d91e as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/d80602b7f2e444f3b16b75a96b89d91e 2024-11-25T05:55:36,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742044_1220 (size=31105) 2024-11-25T05:55:36,802 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/782bbedf211243d49ebe7bf1d645e328 2024-11-25T05:55:36,807 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f2c8004e9247eeca508bdec4ef4bbd6d/C of f2c8004e9247eeca508bdec4ef4bbd6d into d80602b7f2e444f3b16b75a96b89d91e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:36,807 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:36,808 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., storeName=f2c8004e9247eeca508bdec4ef4bbd6d/C, priority=13, startTime=1732514136278; duration=0sec 2024-11-25T05:55:36,808 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:36,808 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f2c8004e9247eeca508bdec4ef4bbd6d:C 2024-11-25T05:55:36,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/48545581810c4236b4b9b2cfa22906fd is 50, key is test_row_0/B:col10/1732514135395/Put/seqid=0 2024-11-25T05:55:36,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742045_1221 (size=12151) 2024-11-25T05:55:36,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:36,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514196872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:36,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:36,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:36,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514196872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:36,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:36,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:36,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514196881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:36,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:36,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514196882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:36,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:36,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:36,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:36,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514196882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:36,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:36,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:37,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:37,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514197182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:37,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:37,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:37,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:37,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:37,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514197182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:37,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:37,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:37,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:37,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:37,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514197191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:37,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:37,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:37,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:37,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514197192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:37,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:37,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:37,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:37,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:37,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:37,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514197192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:37,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:37,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:37,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:37,235 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/48545581810c4236b4b9b2cfa22906fd 2024-11-25T05:55:37,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/e9891d59a7e043f495c27337a93a2b08 is 50, key is test_row_0/C:col10/1732514135395/Put/seqid=0 2024-11-25T05:55:37,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742046_1222 (size=12151) 2024-11-25T05:55:37,251 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/e9891d59a7e043f495c27337a93a2b08 2024-11-25T05:55:37,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/782bbedf211243d49ebe7bf1d645e328 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/782bbedf211243d49ebe7bf1d645e328 2024-11-25T05:55:37,266 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/782bbedf211243d49ebe7bf1d645e328, entries=150, sequenceid=253, filesize=30.4 K 2024-11-25T05:55:37,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/48545581810c4236b4b9b2cfa22906fd as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/48545581810c4236b4b9b2cfa22906fd 2024-11-25T05:55:37,273 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/48545581810c4236b4b9b2cfa22906fd, entries=150, sequenceid=253, filesize=11.9 K 2024-11-25T05:55:37,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/e9891d59a7e043f495c27337a93a2b08 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/e9891d59a7e043f495c27337a93a2b08 2024-11-25T05:55:37,281 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/e9891d59a7e043f495c27337a93a2b08, entries=150, sequenceid=253, filesize=11.9 K 2024-11-25T05:55:37,282 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for f2c8004e9247eeca508bdec4ef4bbd6d in 943ms, sequenceid=253, compaction requested=false 2024-11-25T05:55:37,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:37,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:37,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-25T05:55:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-25T05:55:37,286 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-25T05:55:37,286 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7080 sec 2024-11-25T05:55:37,289 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.7150 sec 2024-11-25T05:55:37,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:37,697 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f2c8004e9247eeca508bdec4ef4bbd6d 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-25T05:55:37,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=A 2024-11-25T05:55:37,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:37,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=B 2024-11-25T05:55:37,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:37,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=C 2024-11-25T05:55:37,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:37,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125e8ebc7aee9774c10964ea819d725bcf2_f2c8004e9247eeca508bdec4ef4bbd6d is 50, key is test_row_0/A:col10/1732514137696/Put/seqid=0 2024-11-25T05:55:37,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742047_1223 (size=12454) 2024-11-25T05:55:37,721 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:55:37,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-25T05:55:37,722 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:55:37,723 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:55:37,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:37,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-25T05:55:37,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514197717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:37,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T05:55:37,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:37,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:37,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:37,729 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:55:37,730 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:55:37,730 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:55:37,730 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125e8ebc7aee9774c10964ea819d725bcf2_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125e8ebc7aee9774c10964ea819d725bcf2_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:37,732 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/be75fbb65e7e40fc878d01b2fb997fe6, store: [table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:37,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:37,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514197721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:37,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:37,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514197721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:37,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:37,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:37,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:37,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:37,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:37,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514197722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:37,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:37,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:37,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:37,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:37,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:37,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:37,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/be75fbb65e7e40fc878d01b2fb997fe6 is 175, key is test_row_0/A:col10/1732514137696/Put/seqid=0 2024-11-25T05:55:37,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514197724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:37,734 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:37,734 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:37,734 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:37,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742048_1224 (size=31255) 2024-11-25T05:55:37,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T05:55:37,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:37,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514197832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:37,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:37,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:37,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:37,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:37,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514197842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:37,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:37,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514197842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:37,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:37,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:37,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:37,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:37,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:37,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:37,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:37,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514197842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:37,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:37,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:37,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:37,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:37,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514197843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:37,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:37,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:37,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:37,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-25T05:55:37,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:37,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:37,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:37,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:37,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:37,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:38,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-25T05:55:38,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:38,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:38,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:38,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:38,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:38,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:38,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T05:55:38,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:38,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514198042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:38,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:38,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:38,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:38,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:38,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514198052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:38,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:38,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:38,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:38,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:38,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514198052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:38,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:38,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:38,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:38,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:38,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514198052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:38,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:38,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:38,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:38,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514198061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:38,139 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=271, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/be75fbb65e7e40fc878d01b2fb997fe6 2024-11-25T05:55:38,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/8126ff08900a42889fc07c1252801a6e is 50, key is test_row_0/B:col10/1732514137696/Put/seqid=0 2024-11-25T05:55:38,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742049_1225 (size=12301) 2024-11-25T05:55:38,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/8126ff08900a42889fc07c1252801a6e 2024-11-25T05:55:38,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/080b6e8cbb124485810a05d94797e100 is 50, key is test_row_0/C:col10/1732514137696/Put/seqid=0 2024-11-25T05:55:38,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742050_1226 (size=12301) 2024-11-25T05:55:38,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-25T05:55:38,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:38,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:38,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:38,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:38,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:38,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:38,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-25T05:55:38,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:38,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:38,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:38,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:38,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:38,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:38,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T05:55:38,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:38,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514198352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:38,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:38,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:38,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:38,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:38,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514198362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:38,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:38,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:38,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:38,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:38,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514198362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:38,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:38,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:38,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:38,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:38,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514198362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:38,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:38,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:38,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:38,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:38,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514198371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:38,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:38,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:38,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:38,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-25T05:55:38,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:38,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:38,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:38,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:38,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:38,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:38,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/080b6e8cbb124485810a05d94797e100 2024-11-25T05:55:38,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/be75fbb65e7e40fc878d01b2fb997fe6 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/be75fbb65e7e40fc878d01b2fb997fe6 2024-11-25T05:55:38,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/be75fbb65e7e40fc878d01b2fb997fe6, entries=150, sequenceid=271, filesize=30.5 K 2024-11-25T05:55:38,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/8126ff08900a42889fc07c1252801a6e as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/8126ff08900a42889fc07c1252801a6e 2024-11-25T05:55:38,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/8126ff08900a42889fc07c1252801a6e, entries=150, sequenceid=271, filesize=12.0 K 2024-11-25T05:55:38,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/080b6e8cbb124485810a05d94797e100 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/080b6e8cbb124485810a05d94797e100 2024-11-25T05:55:38,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/080b6e8cbb124485810a05d94797e100, entries=150, sequenceid=271, filesize=12.0 K 2024-11-25T05:55:38,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for f2c8004e9247eeca508bdec4ef4bbd6d in 913ms, sequenceid=271, compaction requested=true 2024-11-25T05:55:38,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:38,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f2c8004e9247eeca508bdec4ef4bbd6d:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:55:38,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:38,610 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:38,610 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:38,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f2c8004e9247eeca508bdec4ef4bbd6d:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:55:38,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:38,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f2c8004e9247eeca508bdec4ef4bbd6d:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:55:38,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:38,614 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93943 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:38,614 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): f2c8004e9247eeca508bdec4ef4bbd6d/A is initiating minor compaction (all files) 2024-11-25T05:55:38,614 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f2c8004e9247eeca508bdec4ef4bbd6d/A in TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:38,614 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/55b0e8cef01a49d7b370ce670fc4bce0, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/782bbedf211243d49ebe7bf1d645e328, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/be75fbb65e7e40fc878d01b2fb997fe6] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp, totalSize=91.7 K 2024-11-25T05:55:38,614 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:38,614 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:38,614 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): f2c8004e9247eeca508bdec4ef4bbd6d/B is initiating minor compaction (all files) 2024-11-25T05:55:38,614 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. files: [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/55b0e8cef01a49d7b370ce670fc4bce0, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/782bbedf211243d49ebe7bf1d645e328, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/be75fbb65e7e40fc878d01b2fb997fe6] 2024-11-25T05:55:38,614 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f2c8004e9247eeca508bdec4ef4bbd6d/B in TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:38,614 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/cf722fc8d349423e949f1fbd12d0feb7, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/48545581810c4236b4b9b2cfa22906fd, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/8126ff08900a42889fc07c1252801a6e] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp, totalSize=36.2 K 2024-11-25T05:55:38,615 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting cf722fc8d349423e949f1fbd12d0feb7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732514135041 2024-11-25T05:55:38,615 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55b0e8cef01a49d7b370ce670fc4bce0, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732514135041 2024-11-25T05:55:38,617 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 48545581810c4236b4b9b2cfa22906fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732514135391 2024-11-25T05:55:38,617 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 782bbedf211243d49ebe7bf1d645e328, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732514135391 2024-11-25T05:55:38,617 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 8126ff08900a42889fc07c1252801a6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732514136552 2024-11-25T05:55:38,618 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting be75fbb65e7e40fc878d01b2fb997fe6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732514136552 2024-11-25T05:55:38,626 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f2c8004e9247eeca508bdec4ef4bbd6d#B#compaction#194 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:38,626 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/7a005ff0656741968cd0cb44840ff586 is 50, key is test_row_0/B:col10/1732514137696/Put/seqid=0 2024-11-25T05:55:38,642 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:38,648 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411250f7e50b4c9784e4d97ba0cb7fca61635_f2c8004e9247eeca508bdec4ef4bbd6d store=[table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:38,650 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411250f7e50b4c9784e4d97ba0cb7fca61635_f2c8004e9247eeca508bdec4ef4bbd6d, store=[table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:38,650 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411250f7e50b4c9784e4d97ba0cb7fca61635_f2c8004e9247eeca508bdec4ef4bbd6d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:38,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-25T05:55:38,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:38,651 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing f2c8004e9247eeca508bdec4ef4bbd6d 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-25T05:55:38,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=A 2024-11-25T05:55:38,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:38,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=B 2024-11-25T05:55:38,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:38,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=C 2024-11-25T05:55:38,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:38,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742051_1227 (size=12881) 2024-11-25T05:55:38,672 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/7a005ff0656741968cd0cb44840ff586 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/7a005ff0656741968cd0cb44840ff586 2024-11-25T05:55:38,677 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f2c8004e9247eeca508bdec4ef4bbd6d/B of f2c8004e9247eeca508bdec4ef4bbd6d into 7a005ff0656741968cd0cb44840ff586(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:38,677 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:38,677 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., storeName=f2c8004e9247eeca508bdec4ef4bbd6d/B, priority=13, startTime=1732514138610; duration=0sec 2024-11-25T05:55:38,677 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:38,677 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f2c8004e9247eeca508bdec4ef4bbd6d:B 2024-11-25T05:55:38,677 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:38,678 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:38,679 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): f2c8004e9247eeca508bdec4ef4bbd6d/C is initiating minor compaction (all files) 2024-11-25T05:55:38,679 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f2c8004e9247eeca508bdec4ef4bbd6d/C in TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:38,679 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/d80602b7f2e444f3b16b75a96b89d91e, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/e9891d59a7e043f495c27337a93a2b08, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/080b6e8cbb124485810a05d94797e100] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp, totalSize=36.2 K 2024-11-25T05:55:38,679 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting d80602b7f2e444f3b16b75a96b89d91e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732514135041 2024-11-25T05:55:38,680 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting e9891d59a7e043f495c27337a93a2b08, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732514135391 2024-11-25T05:55:38,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125b4d89911c1de418ca4a3bdf614e59a40_f2c8004e9247eeca508bdec4ef4bbd6d is 50, key is test_row_0/A:col10/1732514137716/Put/seqid=0 2024-11-25T05:55:38,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742052_1228 (size=4469) 2024-11-25T05:55:38,688 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f2c8004e9247eeca508bdec4ef4bbd6d#A#compaction#195 average throughput is 0.54 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:38,688 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/2f7735057c1d4001be0e657e92ec5683 is 175, key is test_row_0/A:col10/1732514137696/Put/seqid=0 2024-11-25T05:55:38,689 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 080b6e8cbb124485810a05d94797e100, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732514136552 2024-11-25T05:55:38,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742053_1229 (size=12454) 2024-11-25T05:55:38,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:55:38,706 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f2c8004e9247eeca508bdec4ef4bbd6d#C#compaction#197 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:38,706 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/ac33dbdd291c4c768f50e6e85c3f096f is 50, key is test_row_0/C:col10/1732514137696/Put/seqid=0 2024-11-25T05:55:38,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742054_1230 (size=31835) 2024-11-25T05:55:38,713 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125b4d89911c1de418ca4a3bdf614e59a40_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125b4d89911c1de418ca4a3bdf614e59a40_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:38,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/eb4490b56bba4b0da134e498fc9867f0, store: [table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:38,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/eb4490b56bba4b0da134e498fc9867f0 is 175, key is test_row_0/A:col10/1732514137716/Put/seqid=0 2024-11-25T05:55:38,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742055_1231 (size=12881) 2024-11-25T05:55:38,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742056_1232 (size=31255) 2024-11-25T05:55:38,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:38,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:38,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T05:55:38,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:38,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514198887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:38,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:38,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514198887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:38,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:38,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:38,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:38,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:38,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:38,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:38,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:38,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514198896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:38,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:38,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:38,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:38,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:38,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514198898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:38,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:38,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:38,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:38,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:38,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514198899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:38,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:38,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:38,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:39,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:39,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514199002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:39,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:39,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:39,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:39,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514199003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:39,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:39,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:39,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514199003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:39,004 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:39,004 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:39,004 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:39,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:39,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:39,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:39,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:39,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514199012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:39,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:39,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:39,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:39,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:39,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514199012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:39,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:39,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:39,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:39,114 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/2f7735057c1d4001be0e657e92ec5683 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/2f7735057c1d4001be0e657e92ec5683 2024-11-25T05:55:39,126 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/eb4490b56bba4b0da134e498fc9867f0 2024-11-25T05:55:39,128 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f2c8004e9247eeca508bdec4ef4bbd6d/A of f2c8004e9247eeca508bdec4ef4bbd6d into 2f7735057c1d4001be0e657e92ec5683(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:39,128 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:39,128 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., storeName=f2c8004e9247eeca508bdec4ef4bbd6d/A, priority=13, startTime=1732514138610; duration=0sec 2024-11-25T05:55:39,129 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:39,129 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f2c8004e9247eeca508bdec4ef4bbd6d:A 2024-11-25T05:55:39,135 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/ac33dbdd291c4c768f50e6e85c3f096f as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/ac33dbdd291c4c768f50e6e85c3f096f 2024-11-25T05:55:39,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/22c147781a2f42d1b4f75ac964a54fd5 is 50, key is test_row_0/B:col10/1732514137716/Put/seqid=0 2024-11-25T05:55:39,140 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f2c8004e9247eeca508bdec4ef4bbd6d/C of f2c8004e9247eeca508bdec4ef4bbd6d into ac33dbdd291c4c768f50e6e85c3f096f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:39,140 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:39,140 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., storeName=f2c8004e9247eeca508bdec4ef4bbd6d/C, priority=13, startTime=1732514138610; duration=0sec 2024-11-25T05:55:39,140 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:39,140 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f2c8004e9247eeca508bdec4ef4bbd6d:C 2024-11-25T05:55:39,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742057_1233 (size=12301) 2024-11-25T05:55:39,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:39,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514199212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:39,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:39,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:39,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:39,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:39,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514199212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:39,216 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:39,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:39,216 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:39,216 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:39,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514199212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:39,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:39,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:39,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:39,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:39,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514199222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:39,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:39,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:39,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514199222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:39,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:39,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:39,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:39,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:39,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:39,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:39,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514199522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:39,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:39,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:39,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:39,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:39,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514199522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:39,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:39,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514199522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:39,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:39,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:39,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514199532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:39,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514199532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:39,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:39,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:39,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:39,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:39,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:39,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:39,547 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/22c147781a2f42d1b4f75ac964a54fd5 2024-11-25T05:55:39,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/0e35018b3da444828d5cafebb0e6c7b1 is 50, key is test_row_0/C:col10/1732514137716/Put/seqid=0 2024-11-25T05:55:39,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742058_1234 (size=12301) 2024-11-25T05:55:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T05:55:39,957 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/0e35018b3da444828d5cafebb0e6c7b1 2024-11-25T05:55:39,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/eb4490b56bba4b0da134e498fc9867f0 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/eb4490b56bba4b0da134e498fc9867f0 2024-11-25T05:55:39,968 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/eb4490b56bba4b0da134e498fc9867f0, entries=150, sequenceid=288, filesize=30.5 K 2024-11-25T05:55:39,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/22c147781a2f42d1b4f75ac964a54fd5 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/22c147781a2f42d1b4f75ac964a54fd5 2024-11-25T05:55:39,975 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/22c147781a2f42d1b4f75ac964a54fd5, entries=150, sequenceid=288, filesize=12.0 K 2024-11-25T05:55:39,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/0e35018b3da444828d5cafebb0e6c7b1 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/0e35018b3da444828d5cafebb0e6c7b1 2024-11-25T05:55:39,983 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/0e35018b3da444828d5cafebb0e6c7b1, entries=150, sequenceid=288, filesize=12.0 K 2024-11-25T05:55:39,984 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for f2c8004e9247eeca508bdec4ef4bbd6d in 1334ms, sequenceid=288, compaction requested=false 2024-11-25T05:55:39,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:39,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:39,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-25T05:55:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-25T05:55:39,988 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-25T05:55:39,988 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2550 sec 2024-11-25T05:55:39,991 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.2660 sec 2024-11-25T05:55:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:40,034 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f2c8004e9247eeca508bdec4ef4bbd6d 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-25T05:55:40,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=A 2024-11-25T05:55:40,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:40,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=B 2024-11-25T05:55:40,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:40,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=C 2024-11-25T05:55:40,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:40,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411257c0ebb1e9cb94e47bfeeb9b9a0484ff3_f2c8004e9247eeca508bdec4ef4bbd6d is 50, key is test_row_0/A:col10/1732514138880/Put/seqid=0 2024-11-25T05:55:40,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742059_1235 (size=17534) 2024-11-25T05:55:40,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514200068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:40,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514200069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:40,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514200073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:40,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:40,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514200074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:40,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:40,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:40,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:40,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514200081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:40,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:40,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:40,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514200182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:40,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:40,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514200182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:40,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:40,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:40,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514200182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:40,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:40,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:40,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514200192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:40,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:40,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:40,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514200192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:40,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:40,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:40,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514200393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:40,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:40,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514200393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:40,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:40,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:40,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514200393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:40,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:40,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:40,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:40,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514200402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:40,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:40,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:40,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514200402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:40,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:40,458 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:55:40,461 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411257c0ebb1e9cb94e47bfeeb9b9a0484ff3_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411257c0ebb1e9cb94e47bfeeb9b9a0484ff3_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:40,462 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/a01cbeb0c3434bc999fb3c1068e52f0d, store: [table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:40,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/a01cbeb0c3434bc999fb3c1068e52f0d is 175, key is test_row_0/A:col10/1732514138880/Put/seqid=0 2024-11-25T05:55:40,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742060_1236 (size=48639) 2024-11-25T05:55:40,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:40,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514200703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:40,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:40,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:40,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514200703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:40,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:40,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:40,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514200703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:40,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:40,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:40,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514200712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:40,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:40,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514200712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:40,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:40,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:40,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:40,867 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=309, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/a01cbeb0c3434bc999fb3c1068e52f0d 2024-11-25T05:55:40,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/e30f7f9066d74892b1485541d6333a14 is 50, key is test_row_0/B:col10/1732514138880/Put/seqid=0 2024-11-25T05:55:40,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742061_1237 (size=12301) 2024-11-25T05:55:41,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:41,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514201212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:41,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:41,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:41,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:41,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514201212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:41,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:41,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:41,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:41,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:41,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:41,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514201212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:41,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:41,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:41,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:41,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:41,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514201222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:41,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:41,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514201222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:41,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/e30f7f9066d74892b1485541d6333a14 2024-11-25T05:55:41,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/c77dfb4ece2b4ad683749c859096d0a6 is 50, key is test_row_0/C:col10/1732514138880/Put/seqid=0 2024-11-25T05:55:41,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742062_1238 (size=12301) 2024-11-25T05:55:41,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/c77dfb4ece2b4ad683749c859096d0a6 2024-11-25T05:55:41,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/a01cbeb0c3434bc999fb3c1068e52f0d as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/a01cbeb0c3434bc999fb3c1068e52f0d 2024-11-25T05:55:41,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/a01cbeb0c3434bc999fb3c1068e52f0d, entries=250, sequenceid=309, filesize=47.5 K 2024-11-25T05:55:41,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/e30f7f9066d74892b1485541d6333a14 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/e30f7f9066d74892b1485541d6333a14 2024-11-25T05:55:41,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/e30f7f9066d74892b1485541d6333a14, entries=150, sequenceid=309, filesize=12.0 K 2024-11-25T05:55:41,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/c77dfb4ece2b4ad683749c859096d0a6 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/c77dfb4ece2b4ad683749c859096d0a6 2024-11-25T05:55:41,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/c77dfb4ece2b4ad683749c859096d0a6, entries=150, sequenceid=309, filesize=12.0 K 2024-11-25T05:55:41,308 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for f2c8004e9247eeca508bdec4ef4bbd6d in 1273ms, sequenceid=309, compaction requested=true 2024-11-25T05:55:41,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:41,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f2c8004e9247eeca508bdec4ef4bbd6d:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:55:41,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:41,308 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:41,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f2c8004e9247eeca508bdec4ef4bbd6d:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:55:41,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:41,308 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:41,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f2c8004e9247eeca508bdec4ef4bbd6d:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:55:41,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:41,310 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111729 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:41,310 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:41,310 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): f2c8004e9247eeca508bdec4ef4bbd6d/A is initiating minor compaction (all files) 2024-11-25T05:55:41,310 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): f2c8004e9247eeca508bdec4ef4bbd6d/B is initiating minor compaction (all files) 2024-11-25T05:55:41,310 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f2c8004e9247eeca508bdec4ef4bbd6d/A in TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:41,310 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f2c8004e9247eeca508bdec4ef4bbd6d/B in TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:41,310 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/2f7735057c1d4001be0e657e92ec5683, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/eb4490b56bba4b0da134e498fc9867f0, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/a01cbeb0c3434bc999fb3c1068e52f0d] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp, totalSize=109.1 K 2024-11-25T05:55:41,310 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/7a005ff0656741968cd0cb44840ff586, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/22c147781a2f42d1b4f75ac964a54fd5, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/e30f7f9066d74892b1485541d6333a14] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp, totalSize=36.6 K 2024-11-25T05:55:41,310 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:41,310 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. files: [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/2f7735057c1d4001be0e657e92ec5683, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/eb4490b56bba4b0da134e498fc9867f0, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/a01cbeb0c3434bc999fb3c1068e52f0d] 2024-11-25T05:55:41,311 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a005ff0656741968cd0cb44840ff586, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732514136552 2024-11-25T05:55:41,311 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f7735057c1d4001be0e657e92ec5683, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732514136552 2024-11-25T05:55:41,311 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 22c147781a2f42d1b4f75ac964a54fd5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732514137716 2024-11-25T05:55:41,311 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb4490b56bba4b0da134e498fc9867f0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732514137716 2024-11-25T05:55:41,311 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting e30f7f9066d74892b1485541d6333a14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732514138880 2024-11-25T05:55:41,311 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting a01cbeb0c3434bc999fb3c1068e52f0d, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732514138880 2024-11-25T05:55:41,323 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f2c8004e9247eeca508bdec4ef4bbd6d#B#compaction#203 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:41,324 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/b833361e2fbb4739b6ec6c3f020e4dbe is 50, key is test_row_0/B:col10/1732514138880/Put/seqid=0 2024-11-25T05:55:41,325 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:41,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742063_1239 (size=12983) 2024-11-25T05:55:41,330 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411251875918afe0f4d16824b186c924d4c71_f2c8004e9247eeca508bdec4ef4bbd6d store=[table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:41,332 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411251875918afe0f4d16824b186c924d4c71_f2c8004e9247eeca508bdec4ef4bbd6d, store=[table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:41,332 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411251875918afe0f4d16824b186c924d4c71_f2c8004e9247eeca508bdec4ef4bbd6d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:41,335 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/b833361e2fbb4739b6ec6c3f020e4dbe as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/b833361e2fbb4739b6ec6c3f020e4dbe 2024-11-25T05:55:41,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742064_1240 (size=4469) 2024-11-25T05:55:41,340 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f2c8004e9247eeca508bdec4ef4bbd6d/B of f2c8004e9247eeca508bdec4ef4bbd6d into b833361e2fbb4739b6ec6c3f020e4dbe(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:41,340 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:41,340 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., storeName=f2c8004e9247eeca508bdec4ef4bbd6d/B, priority=13, startTime=1732514141308; duration=0sec 2024-11-25T05:55:41,340 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:41,340 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f2c8004e9247eeca508bdec4ef4bbd6d:B 2024-11-25T05:55:41,340 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:41,342 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:41,342 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): f2c8004e9247eeca508bdec4ef4bbd6d/C is initiating minor compaction (all files) 2024-11-25T05:55:41,342 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f2c8004e9247eeca508bdec4ef4bbd6d/C in TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:41,342 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/ac33dbdd291c4c768f50e6e85c3f096f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/0e35018b3da444828d5cafebb0e6c7b1, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/c77dfb4ece2b4ad683749c859096d0a6] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp, totalSize=36.6 K 2024-11-25T05:55:41,343 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting ac33dbdd291c4c768f50e6e85c3f096f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732514136552 2024-11-25T05:55:41,343 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e35018b3da444828d5cafebb0e6c7b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732514137716 2024-11-25T05:55:41,343 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting c77dfb4ece2b4ad683749c859096d0a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732514138880 2024-11-25T05:55:41,351 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f2c8004e9247eeca508bdec4ef4bbd6d#C#compaction#205 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:41,351 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/c2c9e45be3df4fdf968cb22ddc99eee9 is 50, key is test_row_0/C:col10/1732514138880/Put/seqid=0 2024-11-25T05:55:41,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742065_1241 (size=12983) 2024-11-25T05:55:41,384 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/c2c9e45be3df4fdf968cb22ddc99eee9 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/c2c9e45be3df4fdf968cb22ddc99eee9 2024-11-25T05:55:41,390 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f2c8004e9247eeca508bdec4ef4bbd6d/C of f2c8004e9247eeca508bdec4ef4bbd6d into c2c9e45be3df4fdf968cb22ddc99eee9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:41,390 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:41,390 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., storeName=f2c8004e9247eeca508bdec4ef4bbd6d/C, priority=13, startTime=1732514141308; duration=0sec 2024-11-25T05:55:41,390 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:41,390 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f2c8004e9247eeca508bdec4ef4bbd6d:C 2024-11-25T05:55:41,739 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f2c8004e9247eeca508bdec4ef4bbd6d#A#compaction#204 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:41,739 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/536afe492cd04cf893cc074e9d7829e4 is 175, key is test_row_0/A:col10/1732514138880/Put/seqid=0 2024-11-25T05:55:41,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742066_1242 (size=31937) 2024-11-25T05:55:41,754 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/536afe492cd04cf893cc074e9d7829e4 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/536afe492cd04cf893cc074e9d7829e4 2024-11-25T05:55:41,766 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f2c8004e9247eeca508bdec4ef4bbd6d/A of f2c8004e9247eeca508bdec4ef4bbd6d into 536afe492cd04cf893cc074e9d7829e4(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:41,767 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:41,767 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., storeName=f2c8004e9247eeca508bdec4ef4bbd6d/A, priority=13, startTime=1732514141308; duration=0sec 2024-11-25T05:55:41,767 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:41,767 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f2c8004e9247eeca508bdec4ef4bbd6d:A 2024-11-25T05:55:41,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T05:55:41,893 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:55:41,894 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:55:41,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-25T05:55:41,896 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:55:41,897 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:55:41,897 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:55:41,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-25T05:55:42,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-25T05:55:42,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-25T05:55:42,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:42,050 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing f2c8004e9247eeca508bdec4ef4bbd6d 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-25T05:55:42,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=A 2024-11-25T05:55:42,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:42,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=B 2024-11-25T05:55:42,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:42,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=C 2024-11-25T05:55:42,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:42,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411252a27873994ea4979bf800ebf1c8cbb21_f2c8004e9247eeca508bdec4ef4bbd6d is 50, key is test_row_0/A:col10/1732514140053/Put/seqid=0 2024-11-25T05:55:42,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742067_1243 (size=12454) 2024-11-25T05:55:42,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-25T05:55:42,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:42,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:42,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:42,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514202276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:42,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:42,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514202276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:42,287 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,287 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,287 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:42,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:42,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:42,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514202276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:42,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:42,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:42,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514202282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:42,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:42,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:42,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514202282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:42,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:42,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:42,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514202392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:42,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:42,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:42,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514202392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:42,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:42,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:42,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514202393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:42,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:42,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:42,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514202393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:42,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:42,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:42,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514202393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:42,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:42,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:55:42,473 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411252a27873994ea4979bf800ebf1c8cbb21_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252a27873994ea4979bf800ebf1c8cbb21_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:42,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/ff99ec16159445e593d8a2ea17610bfb, store: [table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:42,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/ff99ec16159445e593d8a2ea17610bfb is 175, key is test_row_0/A:col10/1732514140053/Put/seqid=0 2024-11-25T05:55:42,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742068_1244 (size=31255) 2024-11-25T05:55:42,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-25T05:55:42,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:42,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:42,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514202602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:42,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514202602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:42,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:42,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514202602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:42,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:42,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:42,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514202603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:42,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:42,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:42,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:42,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514202602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:42,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:42,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:42,891 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=327, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/ff99ec16159445e593d8a2ea17610bfb 2024-11-25T05:55:42,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/7478136e998e45ea82a4280a5e40af45 is 50, key is test_row_0/B:col10/1732514140053/Put/seqid=0 2024-11-25T05:55:42,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:42,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514202922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:42,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:42,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514202922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:42,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:42,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:42,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:42,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514202923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:42,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:42,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514202923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:42,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:42,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:42,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514202923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:42,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:42,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:42,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:42,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742069_1245 (size=12301) 2024-11-25T05:55:42,933 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/7478136e998e45ea82a4280a5e40af45 2024-11-25T05:55:42,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/bc20b270375243ebb092a777901ec6d5 is 50, key is test_row_0/C:col10/1732514140053/Put/seqid=0 2024-11-25T05:55:42,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742070_1246 (size=12301) 2024-11-25T05:55:43,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-25T05:55:43,347 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/bc20b270375243ebb092a777901ec6d5 2024-11-25T05:55:43,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/ff99ec16159445e593d8a2ea17610bfb as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/ff99ec16159445e593d8a2ea17610bfb 2024-11-25T05:55:43,355 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/ff99ec16159445e593d8a2ea17610bfb, entries=150, sequenceid=327, filesize=30.5 K 2024-11-25T05:55:43,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/7478136e998e45ea82a4280a5e40af45 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/7478136e998e45ea82a4280a5e40af45 2024-11-25T05:55:43,360 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/7478136e998e45ea82a4280a5e40af45, entries=150, sequenceid=327, filesize=12.0 K 2024-11-25T05:55:43,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/bc20b270375243ebb092a777901ec6d5 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/bc20b270375243ebb092a777901ec6d5 2024-11-25T05:55:43,365 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/bc20b270375243ebb092a777901ec6d5, entries=150, sequenceid=327, filesize=12.0 K 2024-11-25T05:55:43,369 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for f2c8004e9247eeca508bdec4ef4bbd6d in 1319ms, sequenceid=327, compaction requested=false 2024-11-25T05:55:43,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:43,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:43,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-25T05:55:43,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-25T05:55:43,372 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-25T05:55:43,372 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4740 sec 2024-11-25T05:55:43,374 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.4790 sec 2024-11-25T05:55:43,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:43,436 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f2c8004e9247eeca508bdec4ef4bbd6d 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-25T05:55:43,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=A 2024-11-25T05:55:43,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:43,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=B 2024-11-25T05:55:43,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:43,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=C 2024-11-25T05:55:43,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:43,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112532276b408a9d4bc3aa19db9c891ded32_f2c8004e9247eeca508bdec4ef4bbd6d is 50, key is test_row_0/A:col10/1732514143436/Put/seqid=0 2024-11-25T05:55:43,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:43,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514203443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:43,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:43,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:43,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514203444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:43,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742071_1247 (size=17534) 2024-11-25T05:55:43,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:43,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514203444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:43,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:43,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:43,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:43,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:43,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:43,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:43,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514203445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:43,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:43,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:43,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514203445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:43,453 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:43,453 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:43,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:43,453 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:43,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:43,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:43,454 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:55:43,458 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112532276b408a9d4bc3aa19db9c891ded32_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112532276b408a9d4bc3aa19db9c891ded32_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:43,459 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/8556bce52d774a2192e8d584419f39f6, store: [table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:43,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/8556bce52d774a2192e8d584419f39f6 is 175, key is test_row_0/A:col10/1732514143436/Put/seqid=0 2024-11-25T05:55:43,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742072_1248 (size=48639) 2024-11-25T05:55:43,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:43,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514203552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:43,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:43,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:43,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:43,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:43,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514203562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:43,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:43,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:43,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:43,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:43,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514203562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:43,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:43,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:43,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:43,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:43,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514203562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:43,563 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:43,563 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:43,563 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:43,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:43,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514203562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:43,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:43,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:43,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:43,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514203762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:43,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:43,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:43,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:43,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:43,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514203772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:43,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:43,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:43,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:43,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:43,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514203772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:43,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:43,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:43,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:43,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:43,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514203772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:43,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:43,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:43,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:43,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:43,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514203772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:43,777 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:43,777 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:43,777 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:43,868 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=347, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/8556bce52d774a2192e8d584419f39f6 2024-11-25T05:55:43,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/faa67b95fc5b4e5d94b9c6ac79c9880a is 50, key is test_row_0/B:col10/1732514143436/Put/seqid=0 2024-11-25T05:55:43,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742073_1249 (size=12301) 2024-11-25T05:55:44,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-25T05:55:44,052 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:55:44,053 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:55:44,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-25T05:55:44,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T05:55:44,054 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:55:44,055 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:55:44,055 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:55:44,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:44,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514204074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:44,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:44,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:44,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:44,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:44,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514204082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:44,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:44,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514204082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:44,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:44,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:44,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:44,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:44,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514204083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:44,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:44,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:44,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:44,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:44,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514204083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:44,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:44,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:44,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:44,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:44,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:44,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:44,126 INFO [Thread-773 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-773. 2024-11-25T05:55:44,126 DEBUG [Thread-773 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:55:44,126 DEBUG [Thread-773 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:44,126 DEBUG [Thread-773 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:44,126 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:55:44,128 INFO [Thread-777 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-777. 2024-11-25T05:55:44,128 DEBUG [Thread-777 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:55:44,128 DEBUG [Thread-777 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:44,128 DEBUG [Thread-777 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:44,128 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:55:44,129 INFO [Thread-775 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-775. 2024-11-25T05:55:44,129 DEBUG [Thread-775 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:55:44,129 DEBUG [Thread-775 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:44,129 DEBUG [Thread-775 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:44,129 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:55:44,139 INFO [Thread-779 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-779. 2024-11-25T05:55:44,139 DEBUG [Thread-779 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:55:44,139 DEBUG [Thread-779 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:44,139 DEBUG [Thread-779 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:44,139 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:55:44,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T05:55:44,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-25T05:55:44,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:44,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:44,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:44,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:44,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:44,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:44,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/faa67b95fc5b4e5d94b9c6ac79c9880a 2024-11-25T05:55:44,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/d91877a9137c4ec893f976d84778a7b4 is 50, key is test_row_0/C:col10/1732514143436/Put/seqid=0 2024-11-25T05:55:44,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742074_1250 (size=12301) 2024-11-25T05:55:44,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-25T05:55:44,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:44,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:44,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:44,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:44,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:44,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:44,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T05:55:44,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-25T05:55:44,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:44,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:44,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:44,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:44,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:44,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:44,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:44,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39668 deadline: 1732514204583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:44,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:44,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:44,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:44,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:44,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732514204592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39708 deadline: 1732514204592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:44,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39712 deadline: 1732514204592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:44,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732514204592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:55:44,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-25T05:55:44,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:44,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:44,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:44,668 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:44,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:44,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:44,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T05:55:44,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/d91877a9137c4ec893f976d84778a7b4 2024-11-25T05:55:44,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/8556bce52d774a2192e8d584419f39f6 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/8556bce52d774a2192e8d584419f39f6 2024-11-25T05:55:44,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/8556bce52d774a2192e8d584419f39f6, entries=250, sequenceid=347, filesize=47.5 K 2024-11-25T05:55:44,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/faa67b95fc5b4e5d94b9c6ac79c9880a as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/faa67b95fc5b4e5d94b9c6ac79c9880a 2024-11-25T05:55:44,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/faa67b95fc5b4e5d94b9c6ac79c9880a, entries=150, sequenceid=347, filesize=12.0 K 2024-11-25T05:55:44,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/d91877a9137c4ec893f976d84778a7b4 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/d91877a9137c4ec893f976d84778a7b4 2024-11-25T05:55:44,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/d91877a9137c4ec893f976d84778a7b4, entries=150, sequenceid=347, filesize=12.0 K 2024-11-25T05:55:44,725 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for f2c8004e9247eeca508bdec4ef4bbd6d in 1288ms, sequenceid=347, compaction requested=true 2024-11-25T05:55:44,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:44,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f2c8004e9247eeca508bdec4ef4bbd6d:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:55:44,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:44,725 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:44,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f2c8004e9247eeca508bdec4ef4bbd6d:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:55:44,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:44,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f2c8004e9247eeca508bdec4ef4bbd6d:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:55:44,725 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:44,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:44,726 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111831 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:44,726 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:44,726 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): f2c8004e9247eeca508bdec4ef4bbd6d/B is initiating minor compaction (all files) 2024-11-25T05:55:44,726 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): f2c8004e9247eeca508bdec4ef4bbd6d/A is initiating minor compaction (all files) 2024-11-25T05:55:44,726 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f2c8004e9247eeca508bdec4ef4bbd6d/B in TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:44,726 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f2c8004e9247eeca508bdec4ef4bbd6d/A in TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:44,727 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/b833361e2fbb4739b6ec6c3f020e4dbe, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/7478136e998e45ea82a4280a5e40af45, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/faa67b95fc5b4e5d94b9c6ac79c9880a] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp, totalSize=36.7 K 2024-11-25T05:55:44,727 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/536afe492cd04cf893cc074e9d7829e4, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/ff99ec16159445e593d8a2ea17610bfb, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/8556bce52d774a2192e8d584419f39f6] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp, totalSize=109.2 K 2024-11-25T05:55:44,727 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:44,727 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. files: [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/536afe492cd04cf893cc074e9d7829e4, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/ff99ec16159445e593d8a2ea17610bfb, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/8556bce52d774a2192e8d584419f39f6] 2024-11-25T05:55:44,727 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting b833361e2fbb4739b6ec6c3f020e4dbe, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732514138880 2024-11-25T05:55:44,727 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 536afe492cd04cf893cc074e9d7829e4, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732514138880 2024-11-25T05:55:44,727 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7478136e998e45ea82a4280a5e40af45, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732514140053 2024-11-25T05:55:44,727 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting ff99ec16159445e593d8a2ea17610bfb, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732514140053 2024-11-25T05:55:44,727 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting faa67b95fc5b4e5d94b9c6ac79c9880a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732514142246 2024-11-25T05:55:44,727 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 8556bce52d774a2192e8d584419f39f6, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732514142246 2024-11-25T05:55:44,739 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f2c8004e9247eeca508bdec4ef4bbd6d#B#compaction#212 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:44,739 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/871d1cd6a5834d2ca65ae685174956d0 is 50, key is test_row_0/B:col10/1732514143436/Put/seqid=0 2024-11-25T05:55:44,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742075_1251 (size=13085) 2024-11-25T05:55:44,743 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:44,746 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125de91050130984f7296eb98ec02ea1c4d_f2c8004e9247eeca508bdec4ef4bbd6d store=[table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:44,774 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125de91050130984f7296eb98ec02ea1c4d_f2c8004e9247eeca508bdec4ef4bbd6d, store=[table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:44,774 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125de91050130984f7296eb98ec02ea1c4d_f2c8004e9247eeca508bdec4ef4bbd6d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:44,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742076_1252 (size=4469) 2024-11-25T05:55:44,779 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f2c8004e9247eeca508bdec4ef4bbd6d#A#compaction#213 average throughput is 0.68 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:44,780 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/52837ab7f1af40f39b84d48dc167f427 is 175, key is test_row_0/A:col10/1732514143436/Put/seqid=0 2024-11-25T05:55:44,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742077_1253 (size=32039) 2024-11-25T05:55:44,790 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/52837ab7f1af40f39b84d48dc167f427 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/52837ab7f1af40f39b84d48dc167f427 2024-11-25T05:55:44,796 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f2c8004e9247eeca508bdec4ef4bbd6d/A of f2c8004e9247eeca508bdec4ef4bbd6d into 52837ab7f1af40f39b84d48dc167f427(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:44,796 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:44,796 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., storeName=f2c8004e9247eeca508bdec4ef4bbd6d/A, priority=13, startTime=1732514144725; duration=0sec 2024-11-25T05:55:44,796 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:44,796 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f2c8004e9247eeca508bdec4ef4bbd6d:A 2024-11-25T05:55:44,796 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:44,797 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:44,797 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): f2c8004e9247eeca508bdec4ef4bbd6d/C is initiating minor compaction (all files) 2024-11-25T05:55:44,797 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f2c8004e9247eeca508bdec4ef4bbd6d/C in TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:44,798 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/c2c9e45be3df4fdf968cb22ddc99eee9, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/bc20b270375243ebb092a777901ec6d5, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/d91877a9137c4ec893f976d84778a7b4] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp, totalSize=36.7 K 2024-11-25T05:55:44,798 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting c2c9e45be3df4fdf968cb22ddc99eee9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732514138880 2024-11-25T05:55:44,798 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting bc20b270375243ebb092a777901ec6d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732514140053 2024-11-25T05:55:44,799 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting d91877a9137c4ec893f976d84778a7b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732514142246 2024-11-25T05:55:44,815 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f2c8004e9247eeca508bdec4ef4bbd6d#C#compaction#214 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:44,816 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/3061a8c752314675987c39ad6de51ce1 is 50, key is test_row_0/C:col10/1732514143436/Put/seqid=0 2024-11-25T05:55:44,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742078_1254 (size=13085) 2024-11-25T05:55:44,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-25T05:55:44,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:44,821 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing f2c8004e9247eeca508bdec4ef4bbd6d 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-25T05:55:44,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=A 2024-11-25T05:55:44,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:44,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=B 2024-11-25T05:55:44,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:44,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=C 2024-11-25T05:55:44,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:44,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411254f339b71638e4be494416c501e92d998_f2c8004e9247eeca508bdec4ef4bbd6d is 50, key is test_row_0/A:col10/1732514143444/Put/seqid=0 2024-11-25T05:55:44,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742079_1255 (size=12454) 2024-11-25T05:55:45,152 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/871d1cd6a5834d2ca65ae685174956d0 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/871d1cd6a5834d2ca65ae685174956d0 2024-11-25T05:55:45,160 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f2c8004e9247eeca508bdec4ef4bbd6d/B of f2c8004e9247eeca508bdec4ef4bbd6d into 871d1cd6a5834d2ca65ae685174956d0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:45,160 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:45,160 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., storeName=f2c8004e9247eeca508bdec4ef4bbd6d/B, priority=13, startTime=1732514144725; duration=0sec 2024-11-25T05:55:45,160 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:45,160 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f2c8004e9247eeca508bdec4ef4bbd6d:B 2024-11-25T05:55:45,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T05:55:45,226 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/3061a8c752314675987c39ad6de51ce1 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/3061a8c752314675987c39ad6de51ce1 2024-11-25T05:55:45,233 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f2c8004e9247eeca508bdec4ef4bbd6d/C of f2c8004e9247eeca508bdec4ef4bbd6d into 3061a8c752314675987c39ad6de51ce1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:45,233 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:45,233 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., storeName=f2c8004e9247eeca508bdec4ef4bbd6d/C, priority=13, startTime=1732514144725; duration=0sec 2024-11-25T05:55:45,234 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:45,234 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f2c8004e9247eeca508bdec4ef4bbd6d:C 2024-11-25T05:55:45,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:55:45,238 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411254f339b71638e4be494416c501e92d998_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411254f339b71638e4be494416c501e92d998_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:45,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/ad1ff89b553f49afa969475733b515e2, store: [table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:45,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/ad1ff89b553f49afa969475733b515e2 is 175, key is test_row_0/A:col10/1732514143444/Put/seqid=0 2024-11-25T05:55:45,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742080_1256 (size=31255) 2024-11-25T05:55:45,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:45,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. as already flushing 2024-11-25T05:55:45,595 INFO [Thread-762 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-762. 2024-11-25T05:55:45,595 DEBUG [Thread-762 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:55:45,595 DEBUG [Thread-762 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:45,596 DEBUG [Thread-762 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:45,596 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:55:45,610 INFO [Thread-770 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-770. 2024-11-25T05:55:45,610 INFO [Thread-766 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-766. 2024-11-25T05:55:45,610 DEBUG [Thread-766 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:55:45,610 DEBUG [Thread-770 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:55:45,610 DEBUG [Thread-766 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:45,610 DEBUG [Thread-770 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:45,610 DEBUG [Thread-766 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:45,610 DEBUG [Thread-770 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:45,610 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:55:45,610 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:55:45,611 INFO [Thread-768 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-768. 2024-11-25T05:55:45,611 DEBUG [Thread-768 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:55:45,611 DEBUG [Thread-768 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:45,611 DEBUG [Thread-768 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:45,611 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:55:45,613 INFO [Thread-764 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-764. 2024-11-25T05:55:45,614 DEBUG [Thread-764 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:55:45,614 DEBUG [Thread-764 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:45,614 DEBUG [Thread-764 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:45,614 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:55:45,644 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=360, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/ad1ff89b553f49afa969475733b515e2 2024-11-25T05:55:45,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/a00ecc2e77464e1683579fe63613113c is 50, key is test_row_0/B:col10/1732514143444/Put/seqid=0 2024-11-25T05:55:45,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742081_1257 (size=12301) 2024-11-25T05:55:46,058 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/a00ecc2e77464e1683579fe63613113c 2024-11-25T05:55:46,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/2c25fabe55b245dd852039b30db333f6 is 50, key is test_row_0/C:col10/1732514143444/Put/seqid=0 2024-11-25T05:55:46,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742082_1258 (size=12301) 2024-11-25T05:55:46,069 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/2c25fabe55b245dd852039b30db333f6 2024-11-25T05:55:46,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/ad1ff89b553f49afa969475733b515e2 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/ad1ff89b553f49afa969475733b515e2 2024-11-25T05:55:46,080 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/ad1ff89b553f49afa969475733b515e2, entries=150, sequenceid=360, filesize=30.5 K 2024-11-25T05:55:46,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/a00ecc2e77464e1683579fe63613113c as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/a00ecc2e77464e1683579fe63613113c 2024-11-25T05:55:46,085 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/a00ecc2e77464e1683579fe63613113c, entries=150, sequenceid=360, filesize=12.0 K 2024-11-25T05:55:46,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/2c25fabe55b245dd852039b30db333f6 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/2c25fabe55b245dd852039b30db333f6 2024-11-25T05:55:46,089 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/2c25fabe55b245dd852039b30db333f6, entries=150, sequenceid=360, filesize=12.0 K 2024-11-25T05:55:46,090 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=33.54 KB/34350 for f2c8004e9247eeca508bdec4ef4bbd6d in 1269ms, sequenceid=360, compaction requested=false 2024-11-25T05:55:46,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for f2c8004e9247eeca508bdec4ef4bbd6d: 2024-11-25T05:55:46,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:46,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-25T05:55:46,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-25T05:55:46,094 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-25T05:55:46,094 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0370 sec 2024-11-25T05:55:46,096 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.0420 sec 2024-11-25T05:55:46,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T05:55:46,203 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:55:46,203 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-25T05:55:46,203 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-11-25T05:55:46,203 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-25T05:55:46,203 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-25T05:55:46,203 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-25T05:55:46,203 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-25T05:55:46,203 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-25T05:55:46,203 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2751 2024-11-25T05:55:46,203 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2771 2024-11-25T05:55:46,203 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-25T05:55:46,203 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1172 2024-11-25T05:55:46,203 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3516 rows 2024-11-25T05:55:46,203 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1169 2024-11-25T05:55:46,203 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3507 rows 2024-11-25T05:55:46,203 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T05:55:46,203 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T05:55:46,203 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:46,204 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:46,204 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:55:46,205 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-25T05:55:46,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-25T05:55:46,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-25T05:55:46,210 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732514146210"}]},"ts":"1732514146210"} 2024-11-25T05:55:46,213 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-25T05:55:46,213 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-25T05:55:46,215 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T05:55:46,216 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f2c8004e9247eeca508bdec4ef4bbd6d, UNASSIGN}] 2024-11-25T05:55:46,217 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f2c8004e9247eeca508bdec4ef4bbd6d, UNASSIGN 2024-11-25T05:55:46,218 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=f2c8004e9247eeca508bdec4ef4bbd6d, regionState=CLOSING, regionLocation=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:46,219 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f2c8004e9247eeca508bdec4ef4bbd6d, UNASSIGN because future has completed 2024-11-25T05:55:46,219 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T05:55:46,219 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005}] 2024-11-25T05:55:46,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-25T05:55:46,371 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(122): Close f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:46,371 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T05:55:46,371 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1722): Closing f2c8004e9247eeca508bdec4ef4bbd6d, disabling compactions & flushes 2024-11-25T05:55:46,371 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:46,371 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:46,371 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. after waiting 0 ms 2024-11-25T05:55:46,371 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:46,372 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(2902): Flushing f2c8004e9247eeca508bdec4ef4bbd6d 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-25T05:55:46,372 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=A 2024-11-25T05:55:46,372 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:46,372 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=B 2024-11-25T05:55:46,372 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:46,372 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f2c8004e9247eeca508bdec4ef4bbd6d, store=C 2024-11-25T05:55:46,372 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:46,377 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411254045e387770741cda4a264c6651dcfcd_f2c8004e9247eeca508bdec4ef4bbd6d is 50, key is test_row_0/A:col10/1732514145612/Put/seqid=0 2024-11-25T05:55:46,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742083_1259 (size=9914) 2024-11-25T05:55:46,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-25T05:55:46,782 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:55:46,785 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411254045e387770741cda4a264c6651dcfcd_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411254045e387770741cda4a264c6651dcfcd_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:46,786 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/7fc42491453d4128a2728bb3e82b7354, store: [table=TestAcidGuarantees family=A region=f2c8004e9247eeca508bdec4ef4bbd6d] 2024-11-25T05:55:46,787 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/7fc42491453d4128a2728bb3e82b7354 is 175, key is test_row_0/A:col10/1732514145612/Put/seqid=0 2024-11-25T05:55:46,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742084_1260 (size=22561) 2024-11-25T05:55:46,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-25T05:55:47,191 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=370, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/7fc42491453d4128a2728bb3e82b7354 2024-11-25T05:55:47,198 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/a0959b60ce6c42adafa7bf819c55fe03 is 50, key is test_row_0/B:col10/1732514145612/Put/seqid=0 2024-11-25T05:55:47,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742085_1261 (size=9857) 2024-11-25T05:55:47,202 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/a0959b60ce6c42adafa7bf819c55fe03 2024-11-25T05:55:47,211 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/78533b874a1d45caa4b07e5c61acd7f0 is 50, key is test_row_0/C:col10/1732514145612/Put/seqid=0 2024-11-25T05:55:47,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742086_1262 (size=9857) 2024-11-25T05:55:47,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-25T05:55:47,545 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T05:55:47,618 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/78533b874a1d45caa4b07e5c61acd7f0 2024-11-25T05:55:47,623 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/A/7fc42491453d4128a2728bb3e82b7354 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/7fc42491453d4128a2728bb3e82b7354 2024-11-25T05:55:47,628 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/7fc42491453d4128a2728bb3e82b7354, entries=100, sequenceid=370, filesize=22.0 K 2024-11-25T05:55:47,628 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/B/a0959b60ce6c42adafa7bf819c55fe03 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/a0959b60ce6c42adafa7bf819c55fe03 2024-11-25T05:55:47,632 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/a0959b60ce6c42adafa7bf819c55fe03, entries=100, sequenceid=370, filesize=9.6 K 2024-11-25T05:55:47,633 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/.tmp/C/78533b874a1d45caa4b07e5c61acd7f0 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/78533b874a1d45caa4b07e5c61acd7f0 2024-11-25T05:55:47,637 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/78533b874a1d45caa4b07e5c61acd7f0, entries=100, sequenceid=370, filesize=9.6 K 2024-11-25T05:55:47,638 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f2c8004e9247eeca508bdec4ef4bbd6d in 1267ms, sequenceid=370, compaction requested=true 2024-11-25T05:55:47,640 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/5a01127926b0442a85cbae79e2514164, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/685cc15eae92481d98fa917dd88a78dd, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/80f2ee2c7b934f81b4acd7670d3768f2, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/9b6b8449514d4d59833fcd9bbc629ff6, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/0efcbc2e796643dfa5de09370bb55f1b, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/05a05c8f7eb9471e8f6c8a53026be233, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/a35856942c794b30adecbd7c41ae30df, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/b442f0fc0d334651864a29b7d17eba55, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/9e7fd382dbf349a5b4f96d9ea7331b09, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/d09fe96e3da14af18905d78d8567414a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/0e023da5c29b46eda0c3391fa6e9709f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/7a8cf08465854f128704000f05be0a09, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/c98d7a8acadc432aba153c5f2f7dcf18, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/514ead0d11644c4d8b0e89ad3d5a33a5, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/55b0e8cef01a49d7b370ce670fc4bce0, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/782bbedf211243d49ebe7bf1d645e328, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/2f7735057c1d4001be0e657e92ec5683, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/be75fbb65e7e40fc878d01b2fb997fe6, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/eb4490b56bba4b0da134e498fc9867f0, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/a01cbeb0c3434bc999fb3c1068e52f0d, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/536afe492cd04cf893cc074e9d7829e4, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/ff99ec16159445e593d8a2ea17610bfb, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/8556bce52d774a2192e8d584419f39f6] to archive 2024-11-25T05:55:47,641 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T05:55:47,642 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/5a01127926b0442a85cbae79e2514164 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/5a01127926b0442a85cbae79e2514164 2024-11-25T05:55:47,643 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/685cc15eae92481d98fa917dd88a78dd to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/685cc15eae92481d98fa917dd88a78dd 2024-11-25T05:55:47,644 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/80f2ee2c7b934f81b4acd7670d3768f2 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/80f2ee2c7b934f81b4acd7670d3768f2 2024-11-25T05:55:47,645 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/9b6b8449514d4d59833fcd9bbc629ff6 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/9b6b8449514d4d59833fcd9bbc629ff6 2024-11-25T05:55:47,646 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/0efcbc2e796643dfa5de09370bb55f1b to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/0efcbc2e796643dfa5de09370bb55f1b 2024-11-25T05:55:47,647 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/05a05c8f7eb9471e8f6c8a53026be233 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/05a05c8f7eb9471e8f6c8a53026be233 2024-11-25T05:55:47,648 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/a35856942c794b30adecbd7c41ae30df to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/a35856942c794b30adecbd7c41ae30df 2024-11-25T05:55:47,649 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/b442f0fc0d334651864a29b7d17eba55 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/b442f0fc0d334651864a29b7d17eba55 2024-11-25T05:55:47,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/9e7fd382dbf349a5b4f96d9ea7331b09 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/9e7fd382dbf349a5b4f96d9ea7331b09 2024-11-25T05:55:47,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/d09fe96e3da14af18905d78d8567414a to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/d09fe96e3da14af18905d78d8567414a 2024-11-25T05:55:47,651 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/0e023da5c29b46eda0c3391fa6e9709f to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/0e023da5c29b46eda0c3391fa6e9709f 2024-11-25T05:55:47,652 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/7a8cf08465854f128704000f05be0a09 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/7a8cf08465854f128704000f05be0a09 2024-11-25T05:55:47,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/c98d7a8acadc432aba153c5f2f7dcf18 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/c98d7a8acadc432aba153c5f2f7dcf18 2024-11-25T05:55:47,658 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/514ead0d11644c4d8b0e89ad3d5a33a5 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/514ead0d11644c4d8b0e89ad3d5a33a5 2024-11-25T05:55:47,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/55b0e8cef01a49d7b370ce670fc4bce0 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/55b0e8cef01a49d7b370ce670fc4bce0 2024-11-25T05:55:47,660 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/782bbedf211243d49ebe7bf1d645e328 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/782bbedf211243d49ebe7bf1d645e328 2024-11-25T05:55:47,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/2f7735057c1d4001be0e657e92ec5683 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/2f7735057c1d4001be0e657e92ec5683 2024-11-25T05:55:47,662 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/be75fbb65e7e40fc878d01b2fb997fe6 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/be75fbb65e7e40fc878d01b2fb997fe6 2024-11-25T05:55:47,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/eb4490b56bba4b0da134e498fc9867f0 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/eb4490b56bba4b0da134e498fc9867f0 2024-11-25T05:55:47,664 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/a01cbeb0c3434bc999fb3c1068e52f0d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/a01cbeb0c3434bc999fb3c1068e52f0d 2024-11-25T05:55:47,665 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/536afe492cd04cf893cc074e9d7829e4 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/536afe492cd04cf893cc074e9d7829e4 2024-11-25T05:55:47,665 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/ff99ec16159445e593d8a2ea17610bfb to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/ff99ec16159445e593d8a2ea17610bfb 2024-11-25T05:55:47,666 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/8556bce52d774a2192e8d584419f39f6 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/8556bce52d774a2192e8d584419f39f6 2024-11-25T05:55:47,668 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/9b8e964ffa6441869b2318349ed663e8, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/d13afc3dadbe4f5392154c36cd2e516f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/ea57d71e91c44a1f98dbc3a5007a7231, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/693a19c514ae487da2bd1ad541fdb7a8, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/db31d4e994e84addbb4625ede1d6442b, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/520a28ec3efd4749bffbcd40e1a22728, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/95ce0136df784c03a3bb5d1b9b9216b9, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/86f2b20e7d7046d1b19ca9583b3a8e6f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/0df351a772954dfaac2f43e95d56d55e, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/952af1dcda4e48bd91c00454d7876723, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/70e317a771724227ab883a240303a0ea, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/343745c1087c4d86941ef5f977630ceb, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/f86ff5697aac491cb5b841dc0a06b4ec, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/cf722fc8d349423e949f1fbd12d0feb7, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/9ceeb009ef904c26a93584ebf5ef15d1, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/48545581810c4236b4b9b2cfa22906fd, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/7a005ff0656741968cd0cb44840ff586, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/8126ff08900a42889fc07c1252801a6e, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/22c147781a2f42d1b4f75ac964a54fd5, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/b833361e2fbb4739b6ec6c3f020e4dbe, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/e30f7f9066d74892b1485541d6333a14, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/7478136e998e45ea82a4280a5e40af45, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/faa67b95fc5b4e5d94b9c6ac79c9880a] to archive 2024-11-25T05:55:47,669 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T05:55:47,671 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/9b8e964ffa6441869b2318349ed663e8 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/9b8e964ffa6441869b2318349ed663e8 2024-11-25T05:55:47,672 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/d13afc3dadbe4f5392154c36cd2e516f to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/d13afc3dadbe4f5392154c36cd2e516f 2024-11-25T05:55:47,673 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/ea57d71e91c44a1f98dbc3a5007a7231 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/ea57d71e91c44a1f98dbc3a5007a7231 2024-11-25T05:55:47,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/693a19c514ae487da2bd1ad541fdb7a8 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/693a19c514ae487da2bd1ad541fdb7a8 2024-11-25T05:55:47,676 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/db31d4e994e84addbb4625ede1d6442b to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/db31d4e994e84addbb4625ede1d6442b 2024-11-25T05:55:47,677 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/520a28ec3efd4749bffbcd40e1a22728 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/520a28ec3efd4749bffbcd40e1a22728 2024-11-25T05:55:47,677 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/95ce0136df784c03a3bb5d1b9b9216b9 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/95ce0136df784c03a3bb5d1b9b9216b9 2024-11-25T05:55:47,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/86f2b20e7d7046d1b19ca9583b3a8e6f to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/86f2b20e7d7046d1b19ca9583b3a8e6f 2024-11-25T05:55:47,679 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/0df351a772954dfaac2f43e95d56d55e to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/0df351a772954dfaac2f43e95d56d55e 2024-11-25T05:55:47,680 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/952af1dcda4e48bd91c00454d7876723 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/952af1dcda4e48bd91c00454d7876723 2024-11-25T05:55:47,681 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/70e317a771724227ab883a240303a0ea to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/70e317a771724227ab883a240303a0ea 2024-11-25T05:55:47,682 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/343745c1087c4d86941ef5f977630ceb to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/343745c1087c4d86941ef5f977630ceb 2024-11-25T05:55:47,683 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/f86ff5697aac491cb5b841dc0a06b4ec to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/f86ff5697aac491cb5b841dc0a06b4ec 2024-11-25T05:55:47,684 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/cf722fc8d349423e949f1fbd12d0feb7 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/cf722fc8d349423e949f1fbd12d0feb7 2024-11-25T05:55:47,686 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/9ceeb009ef904c26a93584ebf5ef15d1 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/9ceeb009ef904c26a93584ebf5ef15d1 2024-11-25T05:55:47,687 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/48545581810c4236b4b9b2cfa22906fd to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/48545581810c4236b4b9b2cfa22906fd 2024-11-25T05:55:47,689 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/7a005ff0656741968cd0cb44840ff586 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/7a005ff0656741968cd0cb44840ff586 2024-11-25T05:55:47,691 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/8126ff08900a42889fc07c1252801a6e to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/8126ff08900a42889fc07c1252801a6e 2024-11-25T05:55:47,692 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/22c147781a2f42d1b4f75ac964a54fd5 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/22c147781a2f42d1b4f75ac964a54fd5 2024-11-25T05:55:47,693 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/b833361e2fbb4739b6ec6c3f020e4dbe to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/b833361e2fbb4739b6ec6c3f020e4dbe 2024-11-25T05:55:47,695 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/e30f7f9066d74892b1485541d6333a14 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/e30f7f9066d74892b1485541d6333a14 2024-11-25T05:55:47,696 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/7478136e998e45ea82a4280a5e40af45 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/7478136e998e45ea82a4280a5e40af45 2024-11-25T05:55:47,698 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/faa67b95fc5b4e5d94b9c6ac79c9880a to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/faa67b95fc5b4e5d94b9c6ac79c9880a 2024-11-25T05:55:47,702 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/a4131ce91f3d453ab62e0fbc0ea20517, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/765522b74ae74ab2acfd813aebe94b5a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/450328120f824282b10c3213bebf3adc, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/0c3c8f9a08744d1da1d6d5526146d0dd, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/567a5473f5834acfb31d882b0c61e568, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/dca88a81fb8e474892a5160274f56a55, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/2a781b8b83684f659229820d17b1539e, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/ece4f0d968604d17b1068419ec5d3ff7, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/c9919f146577427e9a747a405b034b15, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/ef1f7128407548b5ad5cfd4b46c2f20a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/3aaa18bbf8004b8db496df90670f4f5b, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/62390d50f32545bf87ad53b59a3e8e55, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/e149f426913346a396982f955692f0d7, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/d80602b7f2e444f3b16b75a96b89d91e, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/efbe8c1f8f1c4cf5aa937bbe9e65095c, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/e9891d59a7e043f495c27337a93a2b08, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/ac33dbdd291c4c768f50e6e85c3f096f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/080b6e8cbb124485810a05d94797e100, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/0e35018b3da444828d5cafebb0e6c7b1, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/c2c9e45be3df4fdf968cb22ddc99eee9, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/c77dfb4ece2b4ad683749c859096d0a6, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/bc20b270375243ebb092a777901ec6d5, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/d91877a9137c4ec893f976d84778a7b4] to archive 2024-11-25T05:55:47,704 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T05:55:47,706 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/a4131ce91f3d453ab62e0fbc0ea20517 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/a4131ce91f3d453ab62e0fbc0ea20517 2024-11-25T05:55:47,709 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/765522b74ae74ab2acfd813aebe94b5a to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/765522b74ae74ab2acfd813aebe94b5a 2024-11-25T05:55:47,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/450328120f824282b10c3213bebf3adc to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/450328120f824282b10c3213bebf3adc 2024-11-25T05:55:47,712 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/0c3c8f9a08744d1da1d6d5526146d0dd to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/0c3c8f9a08744d1da1d6d5526146d0dd 2024-11-25T05:55:47,713 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/567a5473f5834acfb31d882b0c61e568 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/567a5473f5834acfb31d882b0c61e568 2024-11-25T05:55:47,715 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/dca88a81fb8e474892a5160274f56a55 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/dca88a81fb8e474892a5160274f56a55 2024-11-25T05:55:47,716 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/2a781b8b83684f659229820d17b1539e to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/2a781b8b83684f659229820d17b1539e 2024-11-25T05:55:47,717 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/ece4f0d968604d17b1068419ec5d3ff7 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/ece4f0d968604d17b1068419ec5d3ff7 2024-11-25T05:55:47,719 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/c9919f146577427e9a747a405b034b15 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/c9919f146577427e9a747a405b034b15 2024-11-25T05:55:47,721 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/ef1f7128407548b5ad5cfd4b46c2f20a to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/ef1f7128407548b5ad5cfd4b46c2f20a 2024-11-25T05:55:47,722 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/3aaa18bbf8004b8db496df90670f4f5b to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/3aaa18bbf8004b8db496df90670f4f5b 2024-11-25T05:55:47,723 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/62390d50f32545bf87ad53b59a3e8e55 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/62390d50f32545bf87ad53b59a3e8e55 2024-11-25T05:55:47,724 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/e149f426913346a396982f955692f0d7 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/e149f426913346a396982f955692f0d7 2024-11-25T05:55:47,725 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/d80602b7f2e444f3b16b75a96b89d91e to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/d80602b7f2e444f3b16b75a96b89d91e 2024-11-25T05:55:47,727 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/efbe8c1f8f1c4cf5aa937bbe9e65095c to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/efbe8c1f8f1c4cf5aa937bbe9e65095c 2024-11-25T05:55:47,728 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/e9891d59a7e043f495c27337a93a2b08 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/e9891d59a7e043f495c27337a93a2b08 2024-11-25T05:55:47,729 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/ac33dbdd291c4c768f50e6e85c3f096f to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/ac33dbdd291c4c768f50e6e85c3f096f 2024-11-25T05:55:47,731 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/080b6e8cbb124485810a05d94797e100 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/080b6e8cbb124485810a05d94797e100 2024-11-25T05:55:47,732 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/0e35018b3da444828d5cafebb0e6c7b1 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/0e35018b3da444828d5cafebb0e6c7b1 2024-11-25T05:55:47,733 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/c2c9e45be3df4fdf968cb22ddc99eee9 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/c2c9e45be3df4fdf968cb22ddc99eee9 2024-11-25T05:55:47,734 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/c77dfb4ece2b4ad683749c859096d0a6 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/c77dfb4ece2b4ad683749c859096d0a6 2024-11-25T05:55:47,736 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/bc20b270375243ebb092a777901ec6d5 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/bc20b270375243ebb092a777901ec6d5 2024-11-25T05:55:47,737 DEBUG [StoreCloser-TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/d91877a9137c4ec893f976d84778a7b4 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/d91877a9137c4ec893f976d84778a7b4 2024-11-25T05:55:47,744 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/recovered.edits/373.seqid, newMaxSeqId=373, maxSeqId=4 2024-11-25T05:55:47,745 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. 2024-11-25T05:55:47,745 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1676): Region close journal for f2c8004e9247eeca508bdec4ef4bbd6d: Waiting for close lock at 1732514146371Running coprocessor pre-close hooks at 1732514146371Disabling compacts and flushes for region at 1732514146371Disabling writes for close at 1732514146371Obtaining lock to block concurrent updates at 1732514146372 (+1 ms)Preparing flush snapshotting stores in f2c8004e9247eeca508bdec4ef4bbd6d at 1732514146372Finished memstore snapshotting TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732514146372Flushing stores of TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d. at 1732514146373 (+1 ms)Flushing f2c8004e9247eeca508bdec4ef4bbd6d/A: creating writer at 1732514146373Flushing mob file f2c8004e9247eeca508bdec4ef4bbd6d/A: appending metadata at 1732514146377 (+4 ms)Flushing mob file f2c8004e9247eeca508bdec4ef4bbd6d/A: closing flushed file at 1732514146377Flushing f2c8004e9247eeca508bdec4ef4bbd6d/A: appending metadata at 1732514146786 (+409 ms)Flushing f2c8004e9247eeca508bdec4ef4bbd6d/A: closing flushed file at 1732514146787 (+1 ms)Flushing f2c8004e9247eeca508bdec4ef4bbd6d/B: creating writer at 1732514147195 (+408 ms)Flushing f2c8004e9247eeca508bdec4ef4bbd6d/B: appending metadata at 1732514147198 (+3 ms)Flushing f2c8004e9247eeca508bdec4ef4bbd6d/B: closing flushed file at 1732514147198Flushing f2c8004e9247eeca508bdec4ef4bbd6d/C: creating writer at 1732514147207 (+9 ms)Flushing f2c8004e9247eeca508bdec4ef4bbd6d/C: appending metadata at 1732514147211 (+4 ms)Flushing f2c8004e9247eeca508bdec4ef4bbd6d/C: closing flushed file at 1732514147211Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77cf2f41: reopening flushed file at 1732514147622 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71952c1d: reopening flushed file at 1732514147628 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b4032df: reopening flushed file at 1732514147633 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f2c8004e9247eeca508bdec4ef4bbd6d in 1267ms, sequenceid=370, compaction requested=true at 1732514147638 (+5 ms)Writing region close event to WAL at 1732514147741 (+103 ms)Running coprocessor post-close hooks at 1732514147745 (+4 ms)Closed at 1732514147745 2024-11-25T05:55:47,747 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(157): Closed f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:47,748 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=f2c8004e9247eeca508bdec4ef4bbd6d, regionState=CLOSED 2024-11-25T05:55:47,749 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 because future has completed 2024-11-25T05:55:47,752 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-25T05:55:47,752 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseRegionProcedure f2c8004e9247eeca508bdec4ef4bbd6d, server=8ef925b832e3,35599,1732514089005 in 1.5310 sec 2024-11-25T05:55:47,754 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-11-25T05:55:47,754 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f2c8004e9247eeca508bdec4ef4bbd6d, UNASSIGN in 1.5360 sec 2024-11-25T05:55:47,757 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-25T05:55:47,757 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5400 sec 2024-11-25T05:55:47,758 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732514147758"}]},"ts":"1732514147758"} 2024-11-25T05:55:47,760 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-25T05:55:47,760 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-25T05:55:47,763 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5560 sec 2024-11-25T05:55:48,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-25T05:55:48,352 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:55:48,353 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-25T05:55:48,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T05:55:48,355 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T05:55:48,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-25T05:55:48,355 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=56, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T05:55:48,358 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:48,361 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A, FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B, FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C, FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/recovered.edits] 2024-11-25T05:55:48,364 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/52837ab7f1af40f39b84d48dc167f427 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/52837ab7f1af40f39b84d48dc167f427 2024-11-25T05:55:48,365 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/7fc42491453d4128a2728bb3e82b7354 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/7fc42491453d4128a2728bb3e82b7354 2024-11-25T05:55:48,367 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/ad1ff89b553f49afa969475733b515e2 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/A/ad1ff89b553f49afa969475733b515e2 2024-11-25T05:55:48,369 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/871d1cd6a5834d2ca65ae685174956d0 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/871d1cd6a5834d2ca65ae685174956d0 2024-11-25T05:55:48,371 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/a00ecc2e77464e1683579fe63613113c to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/a00ecc2e77464e1683579fe63613113c 2024-11-25T05:55:48,373 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/a0959b60ce6c42adafa7bf819c55fe03 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/B/a0959b60ce6c42adafa7bf819c55fe03 2024-11-25T05:55:48,376 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/2c25fabe55b245dd852039b30db333f6 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/2c25fabe55b245dd852039b30db333f6 2024-11-25T05:55:48,379 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/3061a8c752314675987c39ad6de51ce1 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/3061a8c752314675987c39ad6de51ce1 2024-11-25T05:55:48,380 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/78533b874a1d45caa4b07e5c61acd7f0 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/C/78533b874a1d45caa4b07e5c61acd7f0 2024-11-25T05:55:48,383 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/recovered.edits/373.seqid to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d/recovered.edits/373.seqid 2024-11-25T05:55:48,384 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:48,384 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-25T05:55:48,384 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-25T05:55:48,385 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-25T05:55:48,389 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411251ee6d90e831741e08914fd4b625b2af1_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411251ee6d90e831741e08914fd4b625b2af1_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:48,391 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411251f752419b31343138cd848e92d2e4648_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411251f752419b31343138cd848e92d2e4648_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:48,392 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252a27873994ea4979bf800ebf1c8cbb21_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252a27873994ea4979bf800ebf1c8cbb21_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:48,393 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252d48b6bbc38d4c6ead19be20153e6411_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252d48b6bbc38d4c6ead19be20153e6411_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:48,394 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112532276b408a9d4bc3aa19db9c891ded32_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112532276b408a9d4bc3aa19db9c891ded32_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:48,395 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411254045e387770741cda4a264c6651dcfcd_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411254045e387770741cda4a264c6651dcfcd_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:48,396 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411254dd3aa7a831a4396821d9bd8000495dd_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411254dd3aa7a831a4396821d9bd8000495dd_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:48,398 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411254f339b71638e4be494416c501e92d998_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411254f339b71638e4be494416c501e92d998_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:48,399 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112557dbf9726dae4ed5a34ad6f2ad5d6570_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112557dbf9726dae4ed5a34ad6f2ad5d6570_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:48,400 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125751e35a47a044dccaeb461814adf6e20_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125751e35a47a044dccaeb461814adf6e20_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:48,401 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411257c0ebb1e9cb94e47bfeeb9b9a0484ff3_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411257c0ebb1e9cb94e47bfeeb9b9a0484ff3_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:48,402 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411258144ae1a13434e99b6c774e043759289_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411258144ae1a13434e99b6c774e043759289_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:48,403 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125b4d89911c1de418ca4a3bdf614e59a40_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125b4d89911c1de418ca4a3bdf614e59a40_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:48,404 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125bc78ed8ddd074bd68c1c7e224bea176c_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125bc78ed8ddd074bd68c1c7e224bea176c_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:48,405 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125cfb145670883406eb836e14aed4598a3_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125cfb145670883406eb836e14aed4598a3_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:48,406 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125dba0c80ce5484e158cb5be88d18aeae2_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125dba0c80ce5484e158cb5be88d18aeae2_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:48,407 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125de1e67c45fe844b7b5d3146807a83127_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125de1e67c45fe844b7b5d3146807a83127_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:48,408 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125e63fdd6efb0e4634af54a8d0bdc29833_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125e63fdd6efb0e4634af54a8d0bdc29833_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:48,409 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125e8ebc7aee9774c10964ea819d725bcf2_f2c8004e9247eeca508bdec4ef4bbd6d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125e8ebc7aee9774c10964ea819d725bcf2_f2c8004e9247eeca508bdec4ef4bbd6d 2024-11-25T05:55:48,410 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-25T05:55:48,412 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=56, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T05:55:48,415 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-25T05:55:48,417 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-25T05:55:48,418 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=56, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T05:55:48,419 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-25T05:55:48,419 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732514148419"}]},"ts":"9223372036854775807"} 2024-11-25T05:55:48,422 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-25T05:55:48,422 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f2c8004e9247eeca508bdec4ef4bbd6d, NAME => 'TestAcidGuarantees,,1732514120894.f2c8004e9247eeca508bdec4ef4bbd6d.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T05:55:48,423 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-25T05:55:48,423 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732514148423"}]},"ts":"9223372036854775807"} 2024-11-25T05:55:48,425 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-25T05:55:48,426 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=56, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T05:55:48,427 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 73 msec 2024-11-25T05:55:48,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-25T05:55:48,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-25T05:55:48,462 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:55:48,472 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=249 (was 247) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6586672e-597e-41c1-14d9-2bcb3b602f07/cluster_8af6f6a5-ddee-228b-0d29-2ed39e53a991/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6586672e-597e-41c1-14d9-2bcb3b602f07/cluster_8af6f6a5-ddee-228b-0d29-2ed39e53a991/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1220560945_22 at /127.0.0.1:58964 [Waiting for operation #252] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1646263908_22 at /127.0.0.1:54278 [Waiting for operation #708] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=398 (was 351) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8252 (was 7058) - AvailableMemoryMB LEAK? - 2024-11-25T05:55:48,481 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=249, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=398, ProcessCount=11, AvailableMemoryMB=8251 2024-11-25T05:55:48,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T05:55:48,483 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T05:55:48,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-25T05:55:48,485 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T05:55:48,485 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:55:48,485 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 57 2024-11-25T05:55:48,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-25T05:55:48,486 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T05:55:48,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742087_1263 (size=1061) 2024-11-25T05:55:48,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-25T05:55:48,765 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-25T05:55:48,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-25T05:55:48,894 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a 2024-11-25T05:55:48,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742088_1264 (size=53) 2024-11-25T05:55:49,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-25T05:55:49,301 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T05:55:49,301 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f0a5da8d44418b4369e0cb022c813f46, disabling compactions & flushes 2024-11-25T05:55:49,301 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:49,301 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:49,301 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. after waiting 0 ms 2024-11-25T05:55:49,301 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:49,301 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:49,301 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f0a5da8d44418b4369e0cb022c813f46: Waiting for close lock at 1732514149301Disabling compacts and flushes for region at 1732514149301Disabling writes for close at 1732514149301Writing region close event to WAL at 1732514149301Closed at 1732514149301 2024-11-25T05:55:49,302 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T05:55:49,303 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732514149302"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732514149302"}]},"ts":"1732514149302"} 2024-11-25T05:55:49,304 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T05:55:49,305 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T05:55:49,305 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732514149305"}]},"ts":"1732514149305"} 2024-11-25T05:55:49,307 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-25T05:55:49,307 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0a5da8d44418b4369e0cb022c813f46, ASSIGN}] 2024-11-25T05:55:49,308 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0a5da8d44418b4369e0cb022c813f46, ASSIGN 2024-11-25T05:55:49,308 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0a5da8d44418b4369e0cb022c813f46, ASSIGN; state=OFFLINE, location=8ef925b832e3,35599,1732514089005; forceNewPlan=false, retain=false 2024-11-25T05:55:49,459 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=f0a5da8d44418b4369e0cb022c813f46, regionState=OPENING, regionLocation=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:49,461 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0a5da8d44418b4369e0cb022c813f46, ASSIGN because future has completed 2024-11-25T05:55:49,461 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005}] 2024-11-25T05:55:49,616 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:49,616 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} 2024-11-25T05:55:49,616 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f0a5da8d44418b4369e0cb022c813f46 2024-11-25T05:55:49,616 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T05:55:49,616 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for f0a5da8d44418b4369e0cb022c813f46 2024-11-25T05:55:49,616 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for f0a5da8d44418b4369e0cb022c813f46 2024-11-25T05:55:49,618 INFO [StoreOpener-f0a5da8d44418b4369e0cb022c813f46-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f0a5da8d44418b4369e0cb022c813f46 2024-11-25T05:55:49,619 INFO [StoreOpener-f0a5da8d44418b4369e0cb022c813f46-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T05:55:49,619 INFO [StoreOpener-f0a5da8d44418b4369e0cb022c813f46-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0a5da8d44418b4369e0cb022c813f46 columnFamilyName A 2024-11-25T05:55:49,619 DEBUG [StoreOpener-f0a5da8d44418b4369e0cb022c813f46-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:55:49,620 INFO [StoreOpener-f0a5da8d44418b4369e0cb022c813f46-1 {}] regionserver.HStore(327): Store=f0a5da8d44418b4369e0cb022c813f46/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T05:55:49,620 INFO [StoreOpener-f0a5da8d44418b4369e0cb022c813f46-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f0a5da8d44418b4369e0cb022c813f46 2024-11-25T05:55:49,621 INFO [StoreOpener-f0a5da8d44418b4369e0cb022c813f46-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T05:55:49,621 INFO [StoreOpener-f0a5da8d44418b4369e0cb022c813f46-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0a5da8d44418b4369e0cb022c813f46 columnFamilyName B 2024-11-25T05:55:49,621 DEBUG [StoreOpener-f0a5da8d44418b4369e0cb022c813f46-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:55:49,622 INFO [StoreOpener-f0a5da8d44418b4369e0cb022c813f46-1 {}] regionserver.HStore(327): Store=f0a5da8d44418b4369e0cb022c813f46/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T05:55:49,622 INFO [StoreOpener-f0a5da8d44418b4369e0cb022c813f46-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f0a5da8d44418b4369e0cb022c813f46 2024-11-25T05:55:49,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-25T05:55:49,623 INFO [StoreOpener-f0a5da8d44418b4369e0cb022c813f46-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T05:55:49,624 INFO [StoreOpener-f0a5da8d44418b4369e0cb022c813f46-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0a5da8d44418b4369e0cb022c813f46 columnFamilyName C 2024-11-25T05:55:49,624 DEBUG [StoreOpener-f0a5da8d44418b4369e0cb022c813f46-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:55:49,624 INFO [StoreOpener-f0a5da8d44418b4369e0cb022c813f46-1 {}] regionserver.HStore(327): Store=f0a5da8d44418b4369e0cb022c813f46/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T05:55:49,625 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:49,625 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for f0a5da8d44418b4369e0cb022c813f46 2024-11-25T05:55:49,626 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46 2024-11-25T05:55:49,626 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46 2024-11-25T05:55:49,627 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for f0a5da8d44418b4369e0cb022c813f46 2024-11-25T05:55:49,627 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for f0a5da8d44418b4369e0cb022c813f46 2024-11-25T05:55:49,627 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T05:55:49,629 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for f0a5da8d44418b4369e0cb022c813f46 2024-11-25T05:55:49,631 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T05:55:49,632 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened f0a5da8d44418b4369e0cb022c813f46; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64030563, jitterRate=-0.045870259404182434}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T05:55:49,632 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f0a5da8d44418b4369e0cb022c813f46 2024-11-25T05:55:49,633 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for f0a5da8d44418b4369e0cb022c813f46: Running coprocessor pre-open hook at 1732514149617Writing region info on filesystem at 1732514149617Initializing all the Stores at 1732514149617Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732514149618 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732514149618Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732514149618Cleaning up temporary data from old regions at 1732514149627 (+9 ms)Running coprocessor post-open hooks at 1732514149632 (+5 ms)Region opened successfully at 1732514149633 (+1 ms) 2024-11-25T05:55:49,634 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., pid=59, masterSystemTime=1732514149613 2024-11-25T05:55:49,635 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:49,636 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:49,636 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=f0a5da8d44418b4369e0cb022c813f46, regionState=OPEN, openSeqNum=2, regionLocation=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:49,637 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 because future has completed 2024-11-25T05:55:49,639 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-25T05:55:49,640 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; OpenRegionProcedure f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 in 177 msec 2024-11-25T05:55:49,641 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-25T05:55:49,641 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0a5da8d44418b4369e0cb022c813f46, ASSIGN in 333 msec 2024-11-25T05:55:49,642 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T05:55:49,642 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732514149642"}]},"ts":"1732514149642"} 2024-11-25T05:55:49,644 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-25T05:55:49,645 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T05:55:49,646 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1620 sec 2024-11-25T05:55:50,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-25T05:55:50,632 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:55:50,633 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@333e91c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:50,633 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:55:50,634 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:55:50,635 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:55:50,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:55:50,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:55:50,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e301b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:50,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:55:50,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:55:50,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:50,637 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56932, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:55:50,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f49276a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:50,638 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:55:50,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:55:50,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:55:50,640 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53774, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:55:50,641 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T05:55:50,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 8ef925b832e3,41285,1732514088402 2024-11-25T05:55:50,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7fc03a96 2024-11-25T05:55:50,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T05:55:50,643 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56946, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T05:55:50,644 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25146acd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:50,644 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:55:50,645 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:55:50,645 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:55:50,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:55:50,646 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:55:50,646 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@378820c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:50,646 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:55:50,646 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:55:50,646 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:50,647 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56954, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:55:50,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3473593d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:50,648 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391cff66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:50,648 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:55:50,649 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:55:50,649 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:55:50,650 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:55:50,650 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:55:50,650 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31e13635, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:50,650 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:55:50,650 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:55:50,650 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:50,651 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56962, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:55:50,651 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20b97ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:50,652 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@135bd0ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:50,652 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:55:50,652 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:55:50,653 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:55:50,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:55:50,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:55:50,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e96aa5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:50,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:55:50,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:55:50,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:50,655 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56980, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:55:50,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a889c5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:50,656 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4609cead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:50,656 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:55:50,656 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:55:50,657 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:55:50,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:55:50,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:55:50,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d2b4850, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:50,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:55:50,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:55:50,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:50,659 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57000, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:55:50,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2992495, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:50,660 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a2df097, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:50,660 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:55:50,660 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:55:50,661 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:55:50,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:55:50,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:55:50,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43924348, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:50,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:55:50,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:55:50,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:50,663 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:55:50,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68a2dca8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:50,664 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5559acf0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:50,664 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:55:50,664 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:55:50,665 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:55:50,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:55:50,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:55:50,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78f6633b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:50,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:55:50,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:55:50,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:50,667 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57050, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:55:50,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5afbbb21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:50,669 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4f6020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:50,669 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:55:50,669 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:55:50,670 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:55:50,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:55:50,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:55:50,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b0f2a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:50,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:55:50,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:55:50,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:50,672 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57062, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:55:50,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66b12f4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:50,673 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692cba37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:50,673 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:55:50,673 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:55:50,674 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:55:50,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:55:50,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:55:50,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2af5b11d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:50,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:55:50,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:55:50,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:50,675 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57076, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:55:50,676 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2da6fa17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:50,680 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d805c88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:50,681 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:55:50,681 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:55:50,682 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:55:50,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:55:50,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:55:50,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1929517e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:50,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:55:50,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:55:50,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:50,684 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57092, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:55:50,684 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6115f56c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:50,685 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4861c2d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:50,685 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:55:50,685 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:55:50,686 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:55:50,686 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:55:50,686 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:55:50,686 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ac103a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:50,687 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:55:50,687 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:55:50,687 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:55:50,688 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57110, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:55:50,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50a61975, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:55:50,690 DEBUG [Thread-1211 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:55:50,691 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:55:50,691 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:55:50,691 DEBUG [Thread-1226 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:55:50,691 DEBUG [Thread-1224 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:55:50,691 DEBUG [Thread-1222 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:55:50,692 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:55:50,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:55:50,692 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:55:50,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:55:50,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:55:50,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:55:50,693 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53782, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:55:50,694 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53794, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:55:50,694 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53806, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:55:50,694 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53814, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:55:50,694 DEBUG [Thread-1230 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:55:50,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:55:50,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:55:50,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2] 2024-11-25T05:55:50,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2] 2024-11-25T05:55:50,696 DEBUG [Thread-1228 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:55:50,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2] 2024-11-25T05:55:50,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2] 2024-11-25T05:55:50,698 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:55:50,698 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:55:50,698 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:55:50,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2] 2024-11-25T05:55:50,700 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53824, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:55:50,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2] 2024-11-25T05:55:50,710 DEBUG [Thread-1215 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:55:50,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f0a5da8d44418b4369e0cb022c813f46 2024-11-25T05:55:50,715 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0a5da8d44418b4369e0cb022c813f46 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-25T05:55:50,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=A 2024-11-25T05:55:50,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:50,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=B 2024-11-25T05:55:50,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:50,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=C 2024-11-25T05:55:50,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:50,717 DEBUG [Thread-1219 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:55:50,717 DEBUG [Thread-1217 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:55:50,717 DEBUG [Thread-1213 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:55:50,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:55:50,718 DEBUG [Thread-1213 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:55:50,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:55:50,719 DEBUG [Thread-1213 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:55:50,720 DEBUG [Thread-1219 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:55:50,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:55:50,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:55:50,721 DEBUG [Thread-1219 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:55:50,722 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:55:50,723 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:55:50,723 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53848, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:55:50,723 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53854, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:55:50,724 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53832, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:55:50,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-25T05:55:50,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T05:55:50,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2] 2024-11-25T05:55:50,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2] 2024-11-25T05:55:50,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2] 2024-11-25T05:55:50,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2] 2024-11-25T05:55:50,729 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:55:50,730 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:55:50,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:55:50,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/725671e32bb24c1287ffd803aa6c2286 is 50, key is test_row_0/A:col10/1732514150714/Put/seqid=0 2024-11-25T05:55:50,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732514150778, max timestamp=1732514150788Num uniques 859; 2024-11-25T05:55:50,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732514150778, max timestamp=1732514150788Num uniques 859; 2024-11-25T05:55:50,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T05:55:50,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T05:55:50,792 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732514150778, max timestamp=1732514150788Num uniques 859; 2024-11-25T05:55:50,792 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T05:55:50,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742089_1265 (size=16681) 2024-11-25T05:55:50,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T05:55:50,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:50,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53848 deadline: 1732514210841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:50,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:50,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:50,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:50,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:50,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53826 deadline: 1732514210841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:50,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:50,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:50,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:50,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:50,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514210852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:50,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:50,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514210852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:50,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:50,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:50,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:50,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:50,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514210855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:50,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:50,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:50,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:50,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:50,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:50,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:50,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-25T05:55:50,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:50,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:55:50,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:50,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:50,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:50,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:50,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:50,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53848 deadline: 1732514210963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:50,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:50,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:50,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:50,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:50,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53826 deadline: 1732514210964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:50,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:50,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514210964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:50,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:50,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514210964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:50,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:50,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514210964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:51,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-25T05:55:51,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:51,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:55:51,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:51,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:51,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T05:55:51,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:51,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:51,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53848 deadline: 1732514211172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:51,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:51,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:51,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514211172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:51,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:51,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:51,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:51,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:51,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:51,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514211182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:51,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:51,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53826 deadline: 1732514211182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:51,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:51,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:51,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:51,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:51,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:51,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:51,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:51,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514211182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:51,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:51,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:51,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:51,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-25T05:55:51,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:51,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:55:51,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:51,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:51,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:51,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:51,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/725671e32bb24c1287ffd803aa6c2286 2024-11-25T05:55:51,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/ac5ea6159b33403fa06e01d7a221da7c is 50, key is test_row_0/B:col10/1732514150714/Put/seqid=0 2024-11-25T05:55:51,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742090_1266 (size=12001) 2024-11-25T05:55:51,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-25T05:55:51,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:51,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:55:51,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:51,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:51,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T05:55:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:51,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53848 deadline: 1732514211482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:51,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514211482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:51,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:51,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:51,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:51,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:51,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:51,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:51,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514211493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:51,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:51,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514211493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:51,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:51,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:51,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:51,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:51,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:51,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53826 deadline: 1732514211493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:51,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:51,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:51,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:51,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-25T05:55:51,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:51,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:55:51,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:51,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:51,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:51,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/ac5ea6159b33403fa06e01d7a221da7c 2024-11-25T05:55:51,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-25T05:55:51,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:51,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:55:51,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:51,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:51,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:51,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/3c95d4aade4e4bb1921e7e9f06cd0e9b is 50, key is test_row_0/C:col10/1732514150714/Put/seqid=0 2024-11-25T05:55:51,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742091_1267 (size=12001) 2024-11-25T05:55:51,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-25T05:55:51,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:51,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:55:51,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:51,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:51,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T05:55:51,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-25T05:55:51,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:51,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:55:51,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:51,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:51,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:51,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:51,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:51,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53848 deadline: 1732514211992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:51,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:51,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514211993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:51,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:51,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:51,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:51,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:51,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:51,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:52,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:52,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514212002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:52,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:52,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:52,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:52,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514212002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:52,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:52,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:52,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:52,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:52,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:52,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53826 deadline: 1732514212012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:52,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:52,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:52,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:52,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-25T05:55:52,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:52,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:55:52,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:52,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:52,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:52,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:52,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/3c95d4aade4e4bb1921e7e9f06cd0e9b 2024-11-25T05:55:52,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/725671e32bb24c1287ffd803aa6c2286 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/725671e32bb24c1287ffd803aa6c2286 2024-11-25T05:55:52,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/725671e32bb24c1287ffd803aa6c2286, entries=250, sequenceid=11, filesize=16.3 K 2024-11-25T05:55:52,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/ac5ea6159b33403fa06e01d7a221da7c as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/ac5ea6159b33403fa06e01d7a221da7c 2024-11-25T05:55:52,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/ac5ea6159b33403fa06e01d7a221da7c, entries=150, sequenceid=11, filesize=11.7 K 2024-11-25T05:55:52,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/3c95d4aade4e4bb1921e7e9f06cd0e9b as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/3c95d4aade4e4bb1921e7e9f06cd0e9b 2024-11-25T05:55:52,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/3c95d4aade4e4bb1921e7e9f06cd0e9b, entries=150, sequenceid=11, filesize=11.7 K 2024-11-25T05:55:52,160 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for f0a5da8d44418b4369e0cb022c813f46 in 1445ms, sequenceid=11, compaction requested=false 2024-11-25T05:55:52,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:55:52,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-25T05:55:52,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:52,274 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing f0a5da8d44418b4369e0cb022c813f46 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-25T05:55:52,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=A 2024-11-25T05:55:52,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:55:52,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=B 2024-11-25T05:55:52,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:55:52,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=C 2024-11-25T05:55:52,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:55:52,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/63a972f8d78a4687a43f3f465c649e55 is 50, key is test_row_0/A:col10/1732514150839/Put/seqid=0 2024-11-25T05:55:52,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742092_1268 (size=12001) 2024-11-25T05:55:52,702 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/63a972f8d78a4687a43f3f465c649e55 2024-11-25T05:55:52,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/e71b59798f0444a8b757a454ca221f57 is 50, key is test_row_0/B:col10/1732514150839/Put/seqid=0 2024-11-25T05:55:52,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742093_1269 (size=12001) 2024-11-25T05:55:52,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T05:55:53,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f0a5da8d44418b4369e0cb022c813f46 2024-11-25T05:55:53,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:55:53,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:53,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53848 deadline: 1732514213016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:53,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:53,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:53,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514213017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:53,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:53,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:53,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514213018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:53,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:53,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:53,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514213018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:53,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53826 deadline: 1732514213022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:53,026 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_0' in f0a5da8d44418b4369e0cb022c813f46 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:55:53,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:53,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:53,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53848 deadline: 1732514213132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:53,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:53,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514213132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:53,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:53,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:53,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:53,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514213132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:53,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:53,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514213132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:53,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:53,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:53,177 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/e71b59798f0444a8b757a454ca221f57 2024-11-25T05:55:53,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/fdd15f89fa1946d5b306857901f01cd6 is 50, key is test_row_0/C:col10/1732514150839/Put/seqid=0 2024-11-25T05:55:53,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742094_1270 (size=12001) 2024-11-25T05:55:53,272 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T05:55:53,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:53,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53848 deadline: 1732514213342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:53,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:53,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514213342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:53,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:53,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:53,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:53,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514213342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:53,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:53,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:53,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514213342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:53,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:53,596 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/fdd15f89fa1946d5b306857901f01cd6 2024-11-25T05:55:53,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/63a972f8d78a4687a43f3f465c649e55 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/63a972f8d78a4687a43f3f465c649e55 2024-11-25T05:55:53,605 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/63a972f8d78a4687a43f3f465c649e55, entries=150, sequenceid=48, filesize=11.7 K 2024-11-25T05:55:53,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/e71b59798f0444a8b757a454ca221f57 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/e71b59798f0444a8b757a454ca221f57 2024-11-25T05:55:53,611 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/e71b59798f0444a8b757a454ca221f57, entries=150, sequenceid=48, filesize=11.7 K 2024-11-25T05:55:53,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/fdd15f89fa1946d5b306857901f01cd6 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/fdd15f89fa1946d5b306857901f01cd6 2024-11-25T05:55:53,618 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/fdd15f89fa1946d5b306857901f01cd6, entries=150, sequenceid=48, filesize=11.7 K 2024-11-25T05:55:53,618 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for f0a5da8d44418b4369e0cb022c813f46 in 1344ms, sequenceid=48, compaction requested=false 2024-11-25T05:55:53,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:55:53,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:53,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-25T05:55:53,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-25T05:55:53,626 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-25T05:55:53,626 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8890 sec 2024-11-25T05:55:53,630 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 2.9060 sec 2024-11-25T05:55:53,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f0a5da8d44418b4369e0cb022c813f46 2024-11-25T05:55:53,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0a5da8d44418b4369e0cb022c813f46 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-25T05:55:53,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=A 2024-11-25T05:55:53,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:53,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=B 2024-11-25T05:55:53,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:53,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=C 2024-11-25T05:55:53,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:53,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/3757c5f4997b41b2b85301ae5326a570 is 50, key is test_row_0/A:col10/1732514153010/Put/seqid=0 2024-11-25T05:55:53,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742095_1271 (size=12001) 2024-11-25T05:55:53,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/3757c5f4997b41b2b85301ae5326a570 2024-11-25T05:55:53,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:53,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514213685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:53,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:53,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514213687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:53,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:53,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:53,695 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732514153682, max timestamp=1732514153689Num uniques 859; 2024-11-25T05:55:53,695 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732514153682, max timestamp=1732514153689Num uniques 859; 2024-11-25T05:55:53,695 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T05:55:53,695 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T05:55:53,696 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732514153682, max timestamp=1732514153689Num uniques 859; 2024-11-25T05:55:53,696 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T05:55:53,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/a5459840e88d4ace899d01152eaf187b is 50, key is test_row_0/B:col10/1732514153010/Put/seqid=0 2024-11-25T05:55:53,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742096_1272 (size=12001) 2024-11-25T05:55:53,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/a5459840e88d4ace899d01152eaf187b 2024-11-25T05:55:53,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/32db3dbe001e4f0f98c7eeef0a3adb34 is 50, key is test_row_0/C:col10/1732514153010/Put/seqid=0 2024-11-25T05:55:53,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742097_1273 (size=12001) 2024-11-25T05:55:53,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:53,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514213757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:53,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:53,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:53,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53848 deadline: 1732514213766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:53,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:53,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:53,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514213803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:53,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:53,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514213803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:53,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:53,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:53,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:53,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514213872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:53,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:53,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:53,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53848 deadline: 1732514213882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:53,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:53,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:54,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:54,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514214012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:54,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:54,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:54,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514214012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:54,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:54,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:54,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:54,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:54,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:54,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:54,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514214082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:54,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:54,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:54,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:54,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:54,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53848 deadline: 1732514214092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:54,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:54,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:54,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:54,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/32db3dbe001e4f0f98c7eeef0a3adb34 2024-11-25T05:55:54,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/3757c5f4997b41b2b85301ae5326a570 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/3757c5f4997b41b2b85301ae5326a570 2024-11-25T05:55:54,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/3757c5f4997b41b2b85301ae5326a570, entries=150, sequenceid=59, filesize=11.7 K 2024-11-25T05:55:54,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/a5459840e88d4ace899d01152eaf187b as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/a5459840e88d4ace899d01152eaf187b 2024-11-25T05:55:54,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/a5459840e88d4ace899d01152eaf187b, entries=150, sequenceid=59, filesize=11.7 K 2024-11-25T05:55:54,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/32db3dbe001e4f0f98c7eeef0a3adb34 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/32db3dbe001e4f0f98c7eeef0a3adb34 2024-11-25T05:55:54,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/32db3dbe001e4f0f98c7eeef0a3adb34, entries=150, sequenceid=59, filesize=11.7 K 2024-11-25T05:55:54,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for f0a5da8d44418b4369e0cb022c813f46 in 545ms, sequenceid=59, compaction requested=true 2024-11-25T05:55:54,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:55:54,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0a5da8d44418b4369e0cb022c813f46:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:55:54,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:54,200 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:54,200 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:54,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0a5da8d44418b4369e0cb022c813f46:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:55:54,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:54,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0a5da8d44418b4369e0cb022c813f46:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:55:54,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:54,201 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:54,201 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): f0a5da8d44418b4369e0cb022c813f46/B is initiating minor compaction (all files) 2024-11-25T05:55:54,201 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0a5da8d44418b4369e0cb022c813f46/B in TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:54,201 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/ac5ea6159b33403fa06e01d7a221da7c, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/e71b59798f0444a8b757a454ca221f57, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/a5459840e88d4ace899d01152eaf187b] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp, totalSize=35.2 K 2024-11-25T05:55:54,201 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:54,201 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): f0a5da8d44418b4369e0cb022c813f46/A is initiating minor compaction (all files) 2024-11-25T05:55:54,201 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0a5da8d44418b4369e0cb022c813f46/A in TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:54,201 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/725671e32bb24c1287ffd803aa6c2286, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/63a972f8d78a4687a43f3f465c649e55, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/3757c5f4997b41b2b85301ae5326a570] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp, totalSize=39.7 K 2024-11-25T05:55:54,201 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting ac5ea6159b33403fa06e01d7a221da7c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732514150710 2024-11-25T05:55:54,201 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 725671e32bb24c1287ffd803aa6c2286, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732514150701 2024-11-25T05:55:54,201 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting e71b59798f0444a8b757a454ca221f57, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732514150839 2024-11-25T05:55:54,202 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63a972f8d78a4687a43f3f465c649e55, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732514150839 2024-11-25T05:55:54,202 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting a5459840e88d4ace899d01152eaf187b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732514153010 2024-11-25T05:55:54,202 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3757c5f4997b41b2b85301ae5326a570, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732514153010 2024-11-25T05:55:54,214 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0a5da8d44418b4369e0cb022c813f46#B#compaction#230 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:54,214 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/34de775130ae4a4780e6b93a0fe2ffbb is 50, key is test_row_0/B:col10/1732514153010/Put/seqid=0 2024-11-25T05:55:54,227 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0a5da8d44418b4369e0cb022c813f46#A#compaction#231 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:54,228 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/4a26ccad7f38481fa949441676e4f3a5 is 50, key is test_row_0/A:col10/1732514153010/Put/seqid=0 2024-11-25T05:55:54,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742098_1274 (size=12104) 2024-11-25T05:55:54,264 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/34de775130ae4a4780e6b93a0fe2ffbb as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/34de775130ae4a4780e6b93a0fe2ffbb 2024-11-25T05:55:54,269 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0a5da8d44418b4369e0cb022c813f46/B of f0a5da8d44418b4369e0cb022c813f46 into 34de775130ae4a4780e6b93a0fe2ffbb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:54,269 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:55:54,269 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., storeName=f0a5da8d44418b4369e0cb022c813f46/B, priority=13, startTime=1732514154200; duration=0sec 2024-11-25T05:55:54,269 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:54,269 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0a5da8d44418b4369e0cb022c813f46:B 2024-11-25T05:55:54,271 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:54,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742099_1275 (size=12104) 2024-11-25T05:55:54,274 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:54,274 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): f0a5da8d44418b4369e0cb022c813f46/C is initiating minor compaction (all files) 2024-11-25T05:55:54,274 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0a5da8d44418b4369e0cb022c813f46/C in TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:54,274 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/3c95d4aade4e4bb1921e7e9f06cd0e9b, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/fdd15f89fa1946d5b306857901f01cd6, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/32db3dbe001e4f0f98c7eeef0a3adb34] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp, totalSize=35.2 K 2024-11-25T05:55:54,274 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c95d4aade4e4bb1921e7e9f06cd0e9b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732514150710 2024-11-25T05:55:54,275 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting fdd15f89fa1946d5b306857901f01cd6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732514150839 2024-11-25T05:55:54,275 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 32db3dbe001e4f0f98c7eeef0a3adb34, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732514153010 2024-11-25T05:55:54,285 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/4a26ccad7f38481fa949441676e4f3a5 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/4a26ccad7f38481fa949441676e4f3a5 2024-11-25T05:55:54,287 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0a5da8d44418b4369e0cb022c813f46#C#compaction#232 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:54,287 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/0adacd070f57473aa7a2b8d5426ce94d is 50, key is test_row_0/C:col10/1732514153010/Put/seqid=0 2024-11-25T05:55:54,293 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0a5da8d44418b4369e0cb022c813f46/A of f0a5da8d44418b4369e0cb022c813f46 into 4a26ccad7f38481fa949441676e4f3a5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:54,293 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:55:54,293 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., storeName=f0a5da8d44418b4369e0cb022c813f46/A, priority=13, startTime=1732514154199; duration=0sec 2024-11-25T05:55:54,293 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:54,293 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0a5da8d44418b4369e0cb022c813f46:A 2024-11-25T05:55:54,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742100_1276 (size=12104) 2024-11-25T05:55:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f0a5da8d44418b4369e0cb022c813f46 2024-11-25T05:55:54,326 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0a5da8d44418b4369e0cb022c813f46 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-25T05:55:54,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=A 2024-11-25T05:55:54,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-25T05:55:54,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=B 2024-11-25T05:55:54,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-25T05:55:54,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=C 2024-11-25T05:55:54,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-25T05:55:54,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/50da8401543e42689f507f272dd05ddf is 50, key is test_row_0/A:col10/1732514154326/Put/seqid=0 2024-11-25T05:55:54,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742101_1277 (size=14341) 2024-11-25T05:55:54,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:54,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:54,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514214357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:54,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514214357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:54,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:54,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:54,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:54,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:54,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:54,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:54,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:54,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514214392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:54,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:54,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:54,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:54,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:54,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53848 deadline: 1732514214402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:54,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:54,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:54,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:54,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:54,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514214472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:54,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:54,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514214472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:54,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:54,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:54,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:54,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:54,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:54,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:54,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514214682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:54,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:54,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:54,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:54,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514214682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:54,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:54,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:54,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:54,719 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/0adacd070f57473aa7a2b8d5426ce94d as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/0adacd070f57473aa7a2b8d5426ce94d 2024-11-25T05:55:54,726 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0a5da8d44418b4369e0cb022c813f46/C of f0a5da8d44418b4369e0cb022c813f46 into 0adacd070f57473aa7a2b8d5426ce94d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:54,726 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:55:54,726 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., storeName=f0a5da8d44418b4369e0cb022c813f46/C, priority=13, startTime=1732514154200; duration=0sec 2024-11-25T05:55:54,726 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:54,726 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0a5da8d44418b4369e0cb022c813f46:C 2024-11-25T05:55:54,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/50da8401543e42689f507f272dd05ddf 2024-11-25T05:55:54,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/60961a8a5e0748a0bb66e22c5fd93810 is 50, key is test_row_0/B:col10/1732514154326/Put/seqid=0 2024-11-25T05:55:54,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742102_1278 (size=12001) 2024-11-25T05:55:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T05:55:54,892 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:55:54,893 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:55:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-25T05:55:54,894 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:55:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-25T05:55:54,896 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:55:54,896 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:55:54,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514214902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:54,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:54,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:54,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:54,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:54,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53848 deadline: 1732514214912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:54,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:54,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:54,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:54,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514214991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:54,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:54,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:54,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:54,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:54,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514214992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:54,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:54,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:54,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-25T05:55:55,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53826 deadline: 1732514215042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:55,044 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_0' in f0a5da8d44418b4369e0cb022c813f46 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:55:55,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:55,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:55,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:55,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-25T05:55:55,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:55,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:55:55,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:55,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:55,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:55,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/60961a8a5e0748a0bb66e22c5fd93810 2024-11-25T05:55:55,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/0a08a8d6189a451ca118a175e7094a06 is 50, key is test_row_0/C:col10/1732514154326/Put/seqid=0 2024-11-25T05:55:55,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742103_1279 (size=12001) 2024-11-25T05:55:55,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-25T05:55:55,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:55,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:55:55,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:55,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:55,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:55,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:55,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-25T05:55:55,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-25T05:55:55,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:55,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:55:55,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:55,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:55,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:55,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:55,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:55,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514215502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:55,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:55,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:55,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514215502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:55,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:55,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:55,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:55,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:55,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:55,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-25T05:55:55,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:55,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:55:55,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:55,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:55,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:55,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:55,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-25T05:55:55,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/0a08a8d6189a451ca118a175e7094a06 2024-11-25T05:55:55,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/50da8401543e42689f507f272dd05ddf as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/50da8401543e42689f507f272dd05ddf 2024-11-25T05:55:55,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/50da8401543e42689f507f272dd05ddf, entries=200, sequenceid=99, filesize=14.0 K 2024-11-25T05:55:55,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/60961a8a5e0748a0bb66e22c5fd93810 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/60961a8a5e0748a0bb66e22c5fd93810 2024-11-25T05:55:55,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/60961a8a5e0748a0bb66e22c5fd93810, entries=150, sequenceid=99, filesize=11.7 K 2024-11-25T05:55:55,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/0a08a8d6189a451ca118a175e7094a06 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/0a08a8d6189a451ca118a175e7094a06 2024-11-25T05:55:55,595 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/0a08a8d6189a451ca118a175e7094a06, entries=150, sequenceid=99, filesize=11.7 K 2024-11-25T05:55:55,595 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for f0a5da8d44418b4369e0cb022c813f46 in 1269ms, sequenceid=99, compaction requested=false 2024-11-25T05:55:55,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:55:55,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-25T05:55:55,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:55,660 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing f0a5da8d44418b4369e0cb022c813f46 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-25T05:55:55,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=A 2024-11-25T05:55:55,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:55,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=B 2024-11-25T05:55:55,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:55,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=C 2024-11-25T05:55:55,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:55,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/aa008b3cbee34146ae4ced4193f41e4d is 50, key is test_row_0/A:col10/1732514154356/Put/seqid=0 2024-11-25T05:55:55,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742104_1280 (size=12001) 2024-11-25T05:55:55,674 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/aa008b3cbee34146ae4ced4193f41e4d 2024-11-25T05:55:55,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/5d8a4063a47b4a6b8ab8871a2f0767ca is 50, key is test_row_0/B:col10/1732514154356/Put/seqid=0 2024-11-25T05:55:55,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742105_1281 (size=12001) 2024-11-25T05:55:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f0a5da8d44418b4369e0cb022c813f46 2024-11-25T05:55:55,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:55:55,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732514155946, max timestamp=1732514155965Num uniques 859; 2024-11-25T05:55:55,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T05:55:55,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53848 deadline: 1732514215983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:55,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:55,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:55,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:55,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514215986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-25T05:55:56,084 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/5d8a4063a47b4a6b8ab8871a2f0767ca 2024-11-25T05:55:56,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/daf5888e2bed4547a565bf6e88b0e677 is 50, key is test_row_0/C:col10/1732514154356/Put/seqid=0 2024-11-25T05:55:56,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742106_1282 (size=12001) 2024-11-25T05:55:56,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:56,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53848 deadline: 1732514216092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:56,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:56,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:56,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:56,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:56,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514216102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:56,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:56,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:56,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:56,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:56,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53848 deadline: 1732514216302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:56,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:56,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:56,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:56,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:56,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514216312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:56,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:56,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:56,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:56,493 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/daf5888e2bed4547a565bf6e88b0e677 2024-11-25T05:55:56,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/aa008b3cbee34146ae4ced4193f41e4d as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/aa008b3cbee34146ae4ced4193f41e4d 2024-11-25T05:55:56,501 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/aa008b3cbee34146ae4ced4193f41e4d, entries=150, sequenceid=109, filesize=11.7 K 2024-11-25T05:55:56,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/5d8a4063a47b4a6b8ab8871a2f0767ca as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/5d8a4063a47b4a6b8ab8871a2f0767ca 2024-11-25T05:55:56,506 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/5d8a4063a47b4a6b8ab8871a2f0767ca, entries=150, sequenceid=109, filesize=11.7 K 2024-11-25T05:55:56,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/daf5888e2bed4547a565bf6e88b0e677 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/daf5888e2bed4547a565bf6e88b0e677 2024-11-25T05:55:56,511 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/daf5888e2bed4547a565bf6e88b0e677, entries=150, sequenceid=109, filesize=11.7 K 2024-11-25T05:55:56,512 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=138.65 KB/141980 for f0a5da8d44418b4369e0cb022c813f46 in 852ms, sequenceid=109, compaction requested=true 2024-11-25T05:55:56,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:55:56,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:56,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-25T05:55:56,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-25T05:55:56,515 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-25T05:55:56,515 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6170 sec 2024-11-25T05:55:56,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.6220 sec 2024-11-25T05:55:56,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f0a5da8d44418b4369e0cb022c813f46 2024-11-25T05:55:56,517 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0a5da8d44418b4369e0cb022c813f46 3/3 column families, dataSize=145.36 KB heapSize=426.56 KB 2024-11-25T05:55:56,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=A 2024-11-25T05:55:56,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:55:56,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=B 2024-11-25T05:55:56,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:55:56,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=C 2024-11-25T05:55:56,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:55:56,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:56,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514216526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:56,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:56,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:56,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:56,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/e83803859f0f4eec8a22e58eaf07e878 is 50, key is test_row_0/A:col10/1732514155985/Put/seqid=0 2024-11-25T05:55:56,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742107_1283 (size=14541) 2024-11-25T05:55:56,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:56,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514216531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:56,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:56,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:56,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:56,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:56,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53848 deadline: 1732514216611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:56,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:56,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:56,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:56,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:56,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514216622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:56,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:56,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:56,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:56,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:56,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514216642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:56,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:56,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:56,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514216642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:56,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:56,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:56,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:56,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:56,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:56,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:56,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514216851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:56,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:56,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:56,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:56,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:56,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514216852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:56,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:56,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:56,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:56,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/e83803859f0f4eec8a22e58eaf07e878 2024-11-25T05:55:56,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/bc9c85b072224d759c352f578ea393ae is 50, key is test_row_0/B:col10/1732514155985/Put/seqid=0 2024-11-25T05:55:56,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742108_1284 (size=12151) 2024-11-25T05:55:57,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-25T05:55:57,042 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:55:57,043 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:55:57,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-25T05:55:57,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-25T05:55:57,044 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:55:57,045 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:55:57,045 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:55:57,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53848 deadline: 1732514217122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:57,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:57,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:57,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:57,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514217132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:57,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:57,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:57,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-25T05:55:57,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514217161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:57,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514217162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:57,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:57,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:57,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:57,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:57,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:57,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:57,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-25T05:55:57,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:57,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:55:57,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:57,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:57,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:57,346 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/bc9c85b072224d759c352f578ea393ae 2024-11-25T05:55:57,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-25T05:55:57,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:57,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:55:57,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:57,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:57,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:57,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/6f221ca9d5934dd8982be2a11b94d620 is 50, key is test_row_0/C:col10/1732514155985/Put/seqid=0 2024-11-25T05:55:57,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742109_1285 (size=12151) 2024-11-25T05:55:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-25T05:55:57,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-25T05:55:57,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:57,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:55:57,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:57,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:57,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:57,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:57,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-25T05:55:57,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:57,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:55:57,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:57,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:57,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:57,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:57,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-25T05:55:57,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:57,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514217672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:57,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:57,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:57,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514217672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:57,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:57,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:57,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:57,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:57,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:57,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/6f221ca9d5934dd8982be2a11b94d620 2024-11-25T05:55:57,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/e83803859f0f4eec8a22e58eaf07e878 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/e83803859f0f4eec8a22e58eaf07e878 2024-11-25T05:55:57,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/e83803859f0f4eec8a22e58eaf07e878, entries=200, sequenceid=138, filesize=14.2 K 2024-11-25T05:55:57,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/bc9c85b072224d759c352f578ea393ae as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/bc9c85b072224d759c352f578ea393ae 2024-11-25T05:55:57,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/bc9c85b072224d759c352f578ea393ae, entries=150, sequenceid=138, filesize=11.9 K 2024-11-25T05:55:57,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/6f221ca9d5934dd8982be2a11b94d620 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/6f221ca9d5934dd8982be2a11b94d620 2024-11-25T05:55:57,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/6f221ca9d5934dd8982be2a11b94d620, entries=150, sequenceid=138, filesize=11.9 K 2024-11-25T05:55:57,777 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~145.36 KB/148850, heapSize ~426.52 KB/436752, currentSize=33.54 KB/34350 for f0a5da8d44418b4369e0cb022c813f46 in 1260ms, sequenceid=138, compaction requested=true 2024-11-25T05:55:57,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:55:57,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0a5da8d44418b4369e0cb022c813f46:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:55:57,778 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T05:55:57,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:57,778 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T05:55:57,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0a5da8d44418b4369e0cb022c813f46:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:55:57,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:57,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0a5da8d44418b4369e0cb022c813f46:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:55:57,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:57,779 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T05:55:57,779 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52987 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T05:55:57,779 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): f0a5da8d44418b4369e0cb022c813f46/B is initiating minor compaction (all files) 2024-11-25T05:55:57,779 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): f0a5da8d44418b4369e0cb022c813f46/A is initiating minor compaction (all files) 2024-11-25T05:55:57,779 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0a5da8d44418b4369e0cb022c813f46/B in TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:57,779 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0a5da8d44418b4369e0cb022c813f46/A in TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:57,779 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/34de775130ae4a4780e6b93a0fe2ffbb, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/60961a8a5e0748a0bb66e22c5fd93810, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/5d8a4063a47b4a6b8ab8871a2f0767ca, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/bc9c85b072224d759c352f578ea393ae] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp, totalSize=47.1 K 2024-11-25T05:55:57,779 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/4a26ccad7f38481fa949441676e4f3a5, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/50da8401543e42689f507f272dd05ddf, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/aa008b3cbee34146ae4ced4193f41e4d, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/e83803859f0f4eec8a22e58eaf07e878] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp, totalSize=51.7 K 2024-11-25T05:55:57,780 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 34de775130ae4a4780e6b93a0fe2ffbb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732514153010 2024-11-25T05:55:57,780 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a26ccad7f38481fa949441676e4f3a5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732514153010 2024-11-25T05:55:57,780 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50da8401543e42689f507f272dd05ddf, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732514153734 2024-11-25T05:55:57,780 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 60961a8a5e0748a0bb66e22c5fd93810, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732514153734 2024-11-25T05:55:57,781 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa008b3cbee34146ae4ced4193f41e4d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732514154336 2024-11-25T05:55:57,781 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d8a4063a47b4a6b8ab8871a2f0767ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732514154336 2024-11-25T05:55:57,782 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting bc9c85b072224d759c352f578ea393ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732514155981 2024-11-25T05:55:57,782 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting e83803859f0f4eec8a22e58eaf07e878, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732514155978 2024-11-25T05:55:57,792 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0a5da8d44418b4369e0cb022c813f46#A#compaction#242 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:57,792 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/c912f17a31f24676854ad616d8a1f019 is 50, key is test_row_0/A:col10/1732514155985/Put/seqid=0 2024-11-25T05:55:57,792 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0a5da8d44418b4369e0cb022c813f46#B#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:57,793 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/f4f6fb10536642f89511ab9c44c0d113 is 50, key is test_row_0/B:col10/1732514155985/Put/seqid=0 2024-11-25T05:55:57,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742111_1287 (size=12391) 2024-11-25T05:55:57,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742110_1286 (size=12391) 2024-11-25T05:55:57,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-25T05:55:57,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:57,809 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing f0a5da8d44418b4369e0cb022c813f46 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-25T05:55:57,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=A 2024-11-25T05:55:57,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:57,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=B 2024-11-25T05:55:57,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:57,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=C 2024-11-25T05:55:57,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:57,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/155a2edf9cae478280468220a0e3d9e6 is 50, key is test_row_0/A:col10/1732514156524/Put/seqid=0 2024-11-25T05:55:57,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742112_1288 (size=12151) 2024-11-25T05:55:58,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:55:58,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f0a5da8d44418b4369e0cb022c813f46 2024-11-25T05:55:58,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-25T05:55:58,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732514158164, max timestamp=1732514158183Num uniques 859; 2024-11-25T05:55:58,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732514158164, max timestamp=1732514158183Num uniques 859; 2024-11-25T05:55:58,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732514158164, max timestamp=1732514158183Num uniques 859; 2024-11-25T05:55:58,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T05:55:58,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T05:55:58,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T05:55:58,203 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/f4f6fb10536642f89511ab9c44c0d113 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/f4f6fb10536642f89511ab9c44c0d113 2024-11-25T05:55:58,207 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/c912f17a31f24676854ad616d8a1f019 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/c912f17a31f24676854ad616d8a1f019 2024-11-25T05:55:58,208 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0a5da8d44418b4369e0cb022c813f46/B of f0a5da8d44418b4369e0cb022c813f46 into f4f6fb10536642f89511ab9c44c0d113(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:58,208 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:55:58,208 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., storeName=f0a5da8d44418b4369e0cb022c813f46/B, priority=12, startTime=1732514157778; duration=0sec 2024-11-25T05:55:58,208 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:58,208 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0a5da8d44418b4369e0cb022c813f46:B 2024-11-25T05:55:58,208 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T05:55:58,210 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T05:55:58,210 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): f0a5da8d44418b4369e0cb022c813f46/C is initiating minor compaction (all files) 2024-11-25T05:55:58,210 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0a5da8d44418b4369e0cb022c813f46/C in TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:58,210 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/0adacd070f57473aa7a2b8d5426ce94d, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/0a08a8d6189a451ca118a175e7094a06, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/daf5888e2bed4547a565bf6e88b0e677, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/6f221ca9d5934dd8982be2a11b94d620] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp, totalSize=47.1 K 2024-11-25T05:55:58,210 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 0adacd070f57473aa7a2b8d5426ce94d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732514153010 2024-11-25T05:55:58,211 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a08a8d6189a451ca118a175e7094a06, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732514153734 2024-11-25T05:55:58,211 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting daf5888e2bed4547a565bf6e88b0e677, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732514154336 2024-11-25T05:55:58,211 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f221ca9d5934dd8982be2a11b94d620, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732514155981 2024-11-25T05:55:58,213 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0a5da8d44418b4369e0cb022c813f46/A of f0a5da8d44418b4369e0cb022c813f46 into c912f17a31f24676854ad616d8a1f019(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:58,213 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:55:58,213 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., storeName=f0a5da8d44418b4369e0cb022c813f46/A, priority=12, startTime=1732514157778; duration=0sec 2024-11-25T05:55:58,214 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:58,214 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0a5da8d44418b4369e0cb022c813f46:A 2024-11-25T05:55:58,218 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/155a2edf9cae478280468220a0e3d9e6 2024-11-25T05:55:58,221 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0a5da8d44418b4369e0cb022c813f46#C#compaction#245 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:58,222 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/5727d717d8a740e489459d4c78cae913 is 50, key is test_row_0/C:col10/1732514155985/Put/seqid=0 2024-11-25T05:55:58,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/a5eb4dd3d1074e5ca8fc0133afa1c603 is 50, key is test_row_0/B:col10/1732514156524/Put/seqid=0 2024-11-25T05:55:58,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742113_1289 (size=12391) 2024-11-25T05:55:58,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742114_1290 (size=12151) 2024-11-25T05:55:58,235 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/a5eb4dd3d1074e5ca8fc0133afa1c603 2024-11-25T05:55:58,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/46baedff6db34a1eb5fecbd32dc24f60 is 50, key is test_row_0/C:col10/1732514156524/Put/seqid=0 2024-11-25T05:55:58,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742115_1291 (size=12151) 2024-11-25T05:55:58,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:58,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514218242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:58,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732514158224, max timestamp=1732514158247Num uniques 1068; 2024-11-25T05:55:58,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-25T05:55:58,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732514158224, max timestamp=1732514158247Num uniques 1068; 2024-11-25T05:55:58,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-25T05:55:58,251 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732514158224, max timestamp=1732514158247Num uniques 1068; 2024-11-25T05:55:58,251 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-25T05:55:58,349 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732514158296, max timestamp=1732514158345Num uniques 1027; 2024-11-25T05:55:58,349 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-25T05:55:58,349 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732514158296, max timestamp=1732514158345Num uniques 1027; 2024-11-25T05:55:58,349 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-25T05:55:58,349 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732514158296, max timestamp=1732514158345Num uniques 1027; 2024-11-25T05:55:58,349 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-25T05:55:58,394 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732514158374, max timestamp=1732514158390Num uniques 1036; 2024-11-25T05:55:58,394 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-25T05:55:58,394 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732514158374, max timestamp=1732514158390Num uniques 1036; 2024-11-25T05:55:58,394 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732514158374, max timestamp=1732514158390Num uniques 1036; 2024-11-25T05:55:58,394 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-25T05:55:58,394 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-25T05:55:58,452 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732514158413, max timestamp=1732514158449Num uniques 1045; 2024-11-25T05:55:58,452 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-25T05:55:58,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732514158413, max timestamp=1732514158449Num uniques 1045; 2024-11-25T05:55:58,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-25T05:55:58,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732514158413, max timestamp=1732514158449Num uniques 1045; 2024-11-25T05:55:58,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-25T05:55:58,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:58,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53848 deadline: 1732514218498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:58,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:58,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:58,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:58,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:58,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514218499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:58,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:58,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:58,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:58,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:58,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53848 deadline: 1732514218612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:58,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:58,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514218612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:58,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:58,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:58,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:58,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:58,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:58,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:58,632 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/5727d717d8a740e489459d4c78cae913 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/5727d717d8a740e489459d4c78cae913 2024-11-25T05:55:58,641 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0a5da8d44418b4369e0cb022c813f46/C of f0a5da8d44418b4369e0cb022c813f46 into 5727d717d8a740e489459d4c78cae913(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:58,641 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:55:58,641 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., storeName=f0a5da8d44418b4369e0cb022c813f46/C, priority=12, startTime=1732514157778; duration=0sec 2024-11-25T05:55:58,641 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:58,641 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0a5da8d44418b4369e0cb022c813f46:C 2024-11-25T05:55:58,647 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/46baedff6db34a1eb5fecbd32dc24f60 2024-11-25T05:55:58,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/155a2edf9cae478280468220a0e3d9e6 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/155a2edf9cae478280468220a0e3d9e6 2024-11-25T05:55:58,655 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/155a2edf9cae478280468220a0e3d9e6, entries=150, sequenceid=146, filesize=11.9 K 2024-11-25T05:55:58,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/a5eb4dd3d1074e5ca8fc0133afa1c603 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/a5eb4dd3d1074e5ca8fc0133afa1c603 2024-11-25T05:55:58,659 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/a5eb4dd3d1074e5ca8fc0133afa1c603, entries=150, sequenceid=146, filesize=11.9 K 2024-11-25T05:55:58,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/46baedff6db34a1eb5fecbd32dc24f60 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/46baedff6db34a1eb5fecbd32dc24f60 2024-11-25T05:55:58,663 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/46baedff6db34a1eb5fecbd32dc24f60, entries=150, sequenceid=146, filesize=11.9 K 2024-11-25T05:55:58,664 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for f0a5da8d44418b4369e0cb022c813f46 in 855ms, sequenceid=146, compaction requested=false 2024-11-25T05:55:58,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:55:58,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:58,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-25T05:55:58,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-25T05:55:58,667 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-25T05:55:58,667 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6210 sec 2024-11-25T05:55:58,669 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.6250 sec 2024-11-25T05:55:58,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f0a5da8d44418b4369e0cb022c813f46 2024-11-25T05:55:58,685 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0a5da8d44418b4369e0cb022c813f46 3/3 column families, dataSize=154.31 KB heapSize=437.39 KB 2024-11-25T05:55:58,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=A 2024-11-25T05:55:58,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-25T05:55:58,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=B 2024-11-25T05:55:58,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-25T05:55:58,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=C 2024-11-25T05:55:58,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-25T05:55:58,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/ec0062674b7241c792bc94530590ea05 is 50, key is test_row_0/A:col10/1732514158492/Put/seqid=0 2024-11-25T05:55:58,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742116_1292 (size=12151) 2024-11-25T05:55:58,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:58,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514218697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:58,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:58,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:58,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:58,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:58,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514218699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:58,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:58,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:58,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:58,764 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-25T05:55:58,765 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-25T05:55:58,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:58,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514218812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:58,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:58,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:58,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:58,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:58,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514218812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:58,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:58,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:58,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:58,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:58,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53848 deadline: 1732514218821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:58,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:58,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:58,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:58,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:58,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514218822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:58,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:58,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:58,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:59,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:59,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514219022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:59,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:59,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:59,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:59,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:59,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514219022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:59,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:59,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:59,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:59,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:59,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53826 deadline: 1732514219062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:59,064 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_0' in f0a5da8d44418b4369e0cb022c813f46 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:55:59,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:59,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:59,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:59,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/ec0062674b7241c792bc94530590ea05 2024-11-25T05:55:59,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/5b2d1032593841e4a22569864862a69a is 50, key is test_row_0/B:col10/1732514158492/Put/seqid=0 2024-11-25T05:55:59,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742117_1293 (size=12151) 2024-11-25T05:55:59,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:59,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53848 deadline: 1732514219132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:59,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:59,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:59,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:59,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:59,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514219132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:59,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:59,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:59,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:59,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-25T05:55:59,202 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:55:59,203 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:55:59,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-25T05:55:59,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-25T05:55:59,204 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:55:59,205 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:55:59,205 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:55:59,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-25T05:55:59,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:59,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514219332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:59,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:59,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:59,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514219332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:59,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:59,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:59,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:59,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:59,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:59,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-25T05:55:59,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:59,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:55:59,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:59,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:59,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:59,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:59,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/5b2d1032593841e4a22569864862a69a 2024-11-25T05:55:59,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-25T05:55:59,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:59,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:55:59,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:59,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:59,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:59,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:59,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/e6086f3d632c48e6b7a1d8d50d9385a4 is 50, key is test_row_0/C:col10/1732514158492/Put/seqid=0 2024-11-25T05:55:59,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742118_1294 (size=12151) 2024-11-25T05:55:59,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-25T05:55:59,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:59,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53848 deadline: 1732514219642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:59,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:59,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514219642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:59,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:59,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:59,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:59,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:59,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:59,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:59,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-25T05:55:59,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:59,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:55:59,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:59,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:59,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:59,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:59,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-25T05:55:59,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:59,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:55:59,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:59,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:59,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:59,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:55:59,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-25T05:55:59,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:59,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514219842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:59,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:55:59,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:59,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:59,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514219842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:55:59,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:59,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:59,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:55:59,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:55:59,918 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/e6086f3d632c48e6b7a1d8d50d9385a4 2024-11-25T05:55:59,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/ec0062674b7241c792bc94530590ea05 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/ec0062674b7241c792bc94530590ea05 2024-11-25T05:55:59,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/ec0062674b7241c792bc94530590ea05, entries=150, sequenceid=256, filesize=11.9 K 2024-11-25T05:55:59,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/5b2d1032593841e4a22569864862a69a as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/5b2d1032593841e4a22569864862a69a 2024-11-25T05:55:59,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/5b2d1032593841e4a22569864862a69a, entries=150, sequenceid=256, filesize=11.9 K 2024-11-25T05:55:59,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/e6086f3d632c48e6b7a1d8d50d9385a4 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/e6086f3d632c48e6b7a1d8d50d9385a4 2024-11-25T05:55:59,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/e6086f3d632c48e6b7a1d8d50d9385a4, entries=150, sequenceid=256, filesize=11.9 K 2024-11-25T05:55:59,942 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~437.34 KB/447840, currentSize=26.84 KB/27480 for f0a5da8d44418b4369e0cb022c813f46 in 1258ms, sequenceid=256, compaction requested=true 2024-11-25T05:55:59,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:55:59,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0a5da8d44418b4369e0cb022c813f46:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:55:59,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:59,943 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:59,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0a5da8d44418b4369e0cb022c813f46:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:55:59,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:59,943 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:59,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0a5da8d44418b4369e0cb022c813f46:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:55:59,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:59,944 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:59,944 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): f0a5da8d44418b4369e0cb022c813f46/B is initiating minor compaction (all files) 2024-11-25T05:55:59,944 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0a5da8d44418b4369e0cb022c813f46/B in TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:59,944 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/f4f6fb10536642f89511ab9c44c0d113, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/a5eb4dd3d1074e5ca8fc0133afa1c603, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/5b2d1032593841e4a22569864862a69a] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp, totalSize=35.8 K 2024-11-25T05:55:59,944 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:59,944 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): f0a5da8d44418b4369e0cb022c813f46/A is initiating minor compaction (all files) 2024-11-25T05:55:59,944 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0a5da8d44418b4369e0cb022c813f46/A in TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:59,944 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/c912f17a31f24676854ad616d8a1f019, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/155a2edf9cae478280468220a0e3d9e6, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/ec0062674b7241c792bc94530590ea05] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp, totalSize=35.8 K 2024-11-25T05:55:59,945 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting f4f6fb10536642f89511ab9c44c0d113, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732514155981 2024-11-25T05:55:59,945 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting c912f17a31f24676854ad616d8a1f019, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732514155981 2024-11-25T05:55:59,945 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting a5eb4dd3d1074e5ca8fc0133afa1c603, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732514156523 2024-11-25T05:55:59,945 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 155a2edf9cae478280468220a0e3d9e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732514156523 2024-11-25T05:55:59,945 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b2d1032593841e4a22569864862a69a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732514158492 2024-11-25T05:55:59,945 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec0062674b7241c792bc94530590ea05, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732514158492 2024-11-25T05:55:59,954 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0a5da8d44418b4369e0cb022c813f46#B#compaction#251 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:59,955 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/d2629b7545764d46ac0311af0abacf5f is 50, key is test_row_0/B:col10/1732514158492/Put/seqid=0 2024-11-25T05:55:59,958 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0a5da8d44418b4369e0cb022c813f46#A#compaction#252 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:55:59,958 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/6033f65298174e96befe685c1186a29f is 50, key is test_row_0/A:col10/1732514158492/Put/seqid=0 2024-11-25T05:55:59,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742119_1295 (size=12493) 2024-11-25T05:55:59,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-25T05:55:59,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:59,970 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing f0a5da8d44418b4369e0cb022c813f46 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-25T05:55:59,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=A 2024-11-25T05:55:59,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:59,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=B 2024-11-25T05:55:59,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:59,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=C 2024-11-25T05:55:59,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:55:59,975 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/d2629b7545764d46ac0311af0abacf5f as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/d2629b7545764d46ac0311af0abacf5f 2024-11-25T05:55:59,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742120_1296 (size=12493) 2024-11-25T05:55:59,984 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0a5da8d44418b4369e0cb022c813f46/B of f0a5da8d44418b4369e0cb022c813f46 into d2629b7545764d46ac0311af0abacf5f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:59,984 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:55:59,984 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., storeName=f0a5da8d44418b4369e0cb022c813f46/B, priority=13, startTime=1732514159943; duration=0sec 2024-11-25T05:55:59,985 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:55:59,985 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0a5da8d44418b4369e0cb022c813f46:B 2024-11-25T05:55:59,985 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:55:59,985 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/6033f65298174e96befe685c1186a29f as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/6033f65298174e96befe685c1186a29f 2024-11-25T05:55:59,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/27093543131b40e5b40b2f10f1034a62 is 50, key is test_row_0/A:col10/1732514158698/Put/seqid=0 2024-11-25T05:55:59,988 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:55:59,988 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): f0a5da8d44418b4369e0cb022c813f46/C is initiating minor compaction (all files) 2024-11-25T05:55:59,988 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0a5da8d44418b4369e0cb022c813f46/C in TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:55:59,988 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/5727d717d8a740e489459d4c78cae913, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/46baedff6db34a1eb5fecbd32dc24f60, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/e6086f3d632c48e6b7a1d8d50d9385a4] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp, totalSize=35.8 K 2024-11-25T05:55:59,989 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 5727d717d8a740e489459d4c78cae913, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732514155981 2024-11-25T05:55:59,990 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 46baedff6db34a1eb5fecbd32dc24f60, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1732514156523 2024-11-25T05:55:59,991 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting e6086f3d632c48e6b7a1d8d50d9385a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732514158492 2024-11-25T05:55:59,994 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0a5da8d44418b4369e0cb022c813f46/A of f0a5da8d44418b4369e0cb022c813f46 into 6033f65298174e96befe685c1186a29f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:55:59,994 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:55:59,994 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., storeName=f0a5da8d44418b4369e0cb022c813f46/A, priority=13, startTime=1732514159943; duration=0sec 2024-11-25T05:55:59,994 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:55:59,994 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0a5da8d44418b4369e0cb022c813f46:A 2024-11-25T05:56:00,003 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0a5da8d44418b4369e0cb022c813f46#C#compaction#254 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:56:00,003 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/081d283367d742c69408594f1dadb720 is 50, key is test_row_0/C:col10/1732514158492/Put/seqid=0 2024-11-25T05:56:00,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742121_1297 (size=9857) 2024-11-25T05:56:00,008 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/27093543131b40e5b40b2f10f1034a62 2024-11-25T05:56:00,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/b80da0051e8544efb4205de236a3c860 is 50, key is test_row_0/B:col10/1732514158698/Put/seqid=0 2024-11-25T05:56:00,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742122_1298 (size=12493) 2024-11-25T05:56:00,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742123_1299 (size=9857) 2024-11-25T05:56:00,046 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/b80da0051e8544efb4205de236a3c860 2024-11-25T05:56:00,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/69b1926e93e84f66b587b0d48d51f50b is 50, key is test_row_0/C:col10/1732514158698/Put/seqid=0 2024-11-25T05:56:00,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742124_1300 (size=9857) 2024-11-25T05:56:00,082 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/69b1926e93e84f66b587b0d48d51f50b 2024-11-25T05:56:00,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/27093543131b40e5b40b2f10f1034a62 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/27093543131b40e5b40b2f10f1034a62 2024-11-25T05:56:00,095 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/27093543131b40e5b40b2f10f1034a62, entries=100, sequenceid=263, filesize=9.6 K 2024-11-25T05:56:00,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/b80da0051e8544efb4205de236a3c860 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/b80da0051e8544efb4205de236a3c860 2024-11-25T05:56:00,104 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/b80da0051e8544efb4205de236a3c860, entries=100, sequenceid=263, filesize=9.6 K 2024-11-25T05:56:00,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/69b1926e93e84f66b587b0d48d51f50b as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/69b1926e93e84f66b587b0d48d51f50b 2024-11-25T05:56:00,117 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/69b1926e93e84f66b587b0d48d51f50b, entries=100, sequenceid=263, filesize=9.6 K 2024-11-25T05:56:00,118 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for f0a5da8d44418b4369e0cb022c813f46 in 148ms, sequenceid=263, compaction requested=false 2024-11-25T05:56:00,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:56:00,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:00,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-25T05:56:00,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-25T05:56:00,122 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-25T05:56:00,122 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 915 msec 2024-11-25T05:56:00,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 920 msec 2024-11-25T05:56:00,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-25T05:56:00,342 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:56:00,343 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:56:00,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-25T05:56:00,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-25T05:56:00,345 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:56:00,346 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:56:00,346 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:56:00,428 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/081d283367d742c69408594f1dadb720 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/081d283367d742c69408594f1dadb720 2024-11-25T05:56:00,433 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0a5da8d44418b4369e0cb022c813f46/C of f0a5da8d44418b4369e0cb022c813f46 into 081d283367d742c69408594f1dadb720(size=12.2 K), total size for store is 21.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:56:00,433 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:56:00,433 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., storeName=f0a5da8d44418b4369e0cb022c813f46/C, priority=13, startTime=1732514159943; duration=0sec 2024-11-25T05:56:00,433 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:00,433 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0a5da8d44418b4369e0cb022c813f46:C 2024-11-25T05:56:00,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-25T05:56:00,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-25T05:56:00,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:00,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:56:00,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:00,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-25T05:56:00,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-25T05:56:00,513 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-25T05:56:00,513 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 165 msec 2024-11-25T05:56:00,514 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 170 msec 2024-11-25T05:56:00,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-25T05:56:00,663 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:56:00,663 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:56:00,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-25T05:56:00,666 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:56:00,668 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:56:00,668 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:56:00,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-25T05:56:00,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f0a5da8d44418b4369e0cb022c813f46 2024-11-25T05:56:00,701 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0a5da8d44418b4369e0cb022c813f46 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-25T05:56:00,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=A 2024-11-25T05:56:00,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:00,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=B 2024-11-25T05:56:00,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:00,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=C 2024-11-25T05:56:00,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:00,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/c95b23e5fcff48adb3edbf68fb1fbd5e is 50, key is test_row_0/A:col10/1732514160663/Put/seqid=0 2024-11-25T05:56:00,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742125_1301 (size=17181) 2024-11-25T05:56:00,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/c95b23e5fcff48adb3edbf68fb1fbd5e 2024-11-25T05:56:00,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/ac86c0026b674301bba5cf4ac18e2a37 is 50, key is test_row_0/B:col10/1732514160663/Put/seqid=0 2024-11-25T05:56:00,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742126_1302 (size=12301) 2024-11-25T05:56:00,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/ac86c0026b674301bba5cf4ac18e2a37 2024-11-25T05:56:00,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/428e3cf0306447c5bfbe879d51cb2fe9 is 50, key is test_row_0/C:col10/1732514160663/Put/seqid=0 2024-11-25T05:56:00,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742127_1303 (size=12301) 2024-11-25T05:56:00,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/428e3cf0306447c5bfbe879d51cb2fe9 2024-11-25T05:56:00,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/c95b23e5fcff48adb3edbf68fb1fbd5e as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/c95b23e5fcff48adb3edbf68fb1fbd5e 2024-11-25T05:56:00,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/c95b23e5fcff48adb3edbf68fb1fbd5e, entries=250, sequenceid=278, filesize=16.8 K 2024-11-25T05:56:00,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/ac86c0026b674301bba5cf4ac18e2a37 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/ac86c0026b674301bba5cf4ac18e2a37 2024-11-25T05:56:00,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732514160723, max timestamp=1732514160749Num uniques 859; 2024-11-25T05:56:00,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732514160723, max timestamp=1732514160749Num uniques 859; 2024-11-25T05:56:00,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T05:56:00,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T05:56:00,755 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732514160723, max timestamp=1732514160752Num uniques 859; 2024-11-25T05:56:00,755 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T05:56:00,756 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/ac86c0026b674301bba5cf4ac18e2a37, entries=150, sequenceid=278, filesize=12.0 K 2024-11-25T05:56:00,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/428e3cf0306447c5bfbe879d51cb2fe9 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/428e3cf0306447c5bfbe879d51cb2fe9 2024-11-25T05:56:00,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/428e3cf0306447c5bfbe879d51cb2fe9, entries=150, sequenceid=278, filesize=12.0 K 2024-11-25T05:56:00,762 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=46.96 KB/48090 for f0a5da8d44418b4369e0cb022c813f46 in 61ms, sequenceid=278, compaction requested=true 2024-11-25T05:56:00,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:56:00,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0a5da8d44418b4369e0cb022c813f46:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:56:00,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:00,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0a5da8d44418b4369e0cb022c813f46:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:56:00,762 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:56:00,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:00,762 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:56:00,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0a5da8d44418b4369e0cb022c813f46:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:56:00,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:56:00,763 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39531 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:56:00,763 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34651 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:56:00,763 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): f0a5da8d44418b4369e0cb022c813f46/B is initiating minor compaction (all files) 2024-11-25T05:56:00,763 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): f0a5da8d44418b4369e0cb022c813f46/A is initiating minor compaction (all files) 2024-11-25T05:56:00,763 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0a5da8d44418b4369e0cb022c813f46/A in TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:00,763 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/6033f65298174e96befe685c1186a29f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/27093543131b40e5b40b2f10f1034a62, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/c95b23e5fcff48adb3edbf68fb1fbd5e] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp, totalSize=38.6 K 2024-11-25T05:56:00,763 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0a5da8d44418b4369e0cb022c813f46/B in TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:00,763 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/d2629b7545764d46ac0311af0abacf5f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/b80da0051e8544efb4205de236a3c860, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/ac86c0026b674301bba5cf4ac18e2a37] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp, totalSize=33.8 K 2024-11-25T05:56:00,764 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting d2629b7545764d46ac0311af0abacf5f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732514158492 2024-11-25T05:56:00,764 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6033f65298174e96befe685c1186a29f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732514158492 2024-11-25T05:56:00,764 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting b80da0051e8544efb4205de236a3c860, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732514158689 2024-11-25T05:56:00,764 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27093543131b40e5b40b2f10f1034a62, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732514158689 2024-11-25T05:56:00,764 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting c95b23e5fcff48adb3edbf68fb1fbd5e, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732514160663 2024-11-25T05:56:00,764 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting ac86c0026b674301bba5cf4ac18e2a37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732514160663 2024-11-25T05:56:00,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f0a5da8d44418b4369e0cb022c813f46 2024-11-25T05:56:00,765 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0a5da8d44418b4369e0cb022c813f46 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-11-25T05:56:00,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=A 2024-11-25T05:56:00,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:56:00,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=B 2024-11-25T05:56:00,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:56:00,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=C 2024-11-25T05:56:00,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:56:00,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-25T05:56:00,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/2f37435177d84f89ac9047b08c8c614a is 50, key is test_row_0/A:col10/1732514160758/Put/seqid=0 2024-11-25T05:56:00,779 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0a5da8d44418b4369e0cb022c813f46#B#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:56:00,779 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/c12eb53113144272bc6373ac7d0680ed is 50, key is test_row_0/B:col10/1732514160663/Put/seqid=0 2024-11-25T05:56:00,783 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0a5da8d44418b4369e0cb022c813f46#A#compaction#262 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:56:00,783 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/51da812ae12a4385b17cd9d8a5f15acf is 50, key is test_row_0/A:col10/1732514160663/Put/seqid=0 2024-11-25T05:56:00,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742129_1305 (size=12745) 2024-11-25T05:56:00,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742128_1304 (size=14741) 2024-11-25T05:56:00,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/2f37435177d84f89ac9047b08c8c614a 2024-11-25T05:56:00,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742130_1306 (size=12745) 2024-11-25T05:56:00,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/37629070169240e6ba441ac5c3369861 is 50, key is test_row_0/B:col10/1732514160758/Put/seqid=0 2024-11-25T05:56:00,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742131_1307 (size=12301) 2024-11-25T05:56:00,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/37629070169240e6ba441ac5c3369861 2024-11-25T05:56:00,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732514160794, max timestamp=1732514160820Num uniques 859; 2024-11-25T05:56:00,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732514160794, max timestamp=1732514160820Num uniques 859; 2024-11-25T05:56:00,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T05:56:00,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T05:56:00,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732514160794, max timestamp=1732514160820Num uniques 859; 2024-11-25T05:56:00,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T05:56:00,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-25T05:56:00,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:00,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:56:00,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:00,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:00,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:00,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:00,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/e06509eb75de4f30b3e640f0f7990f29 is 50, key is test_row_0/C:col10/1732514160758/Put/seqid=0 2024-11-25T05:56:00,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742132_1308 (size=12301) 2024-11-25T05:56:00,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:00,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514220868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:00,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:00,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514220870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:00,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:00,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514220870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:00,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:00,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:00,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:00,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:00,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53848 deadline: 1732514220875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:00,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:00,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:00,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:00,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-25T05:56:00,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:00,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:56:00,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:00,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:00,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:00,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:00,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-25T05:56:00,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:00,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514220982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:00,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:00,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:00,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:00,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:00,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514220982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:00,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:00,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:00,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:00,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:00,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514220983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:00,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:00,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:00,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:00,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:00,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53848 deadline: 1732514220992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:00,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:00,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:00,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:01,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-25T05:56:01,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:01,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:56:01,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:01,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:01,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:01,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:01,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:01,195 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/c12eb53113144272bc6373ac7d0680ed as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/c12eb53113144272bc6373ac7d0680ed 2024-11-25T05:56:01,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514221192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:01,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:01,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:01,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:01,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514221192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:01,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:01,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:01,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:01,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:01,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:01,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514221192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:01,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:01,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:01,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:01,200 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0a5da8d44418b4369e0cb022c813f46/B of f0a5da8d44418b4369e0cb022c813f46 into c12eb53113144272bc6373ac7d0680ed(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:56:01,200 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:56:01,200 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., storeName=f0a5da8d44418b4369e0cb022c813f46/B, priority=13, startTime=1732514160762; duration=0sec 2024-11-25T05:56:01,200 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:56:01,200 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0a5da8d44418b4369e0cb022c813f46:B 2024-11-25T05:56:01,200 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:56:01,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:01,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53848 deadline: 1732514221202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:01,206 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/51da812ae12a4385b17cd9d8a5f15acf as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/51da812ae12a4385b17cd9d8a5f15acf 2024-11-25T05:56:01,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:01,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:01,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:01,206 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34651 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:56:01,206 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): f0a5da8d44418b4369e0cb022c813f46/C is initiating minor compaction (all files) 2024-11-25T05:56:01,206 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0a5da8d44418b4369e0cb022c813f46/C in TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:01,206 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/081d283367d742c69408594f1dadb720, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/69b1926e93e84f66b587b0d48d51f50b, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/428e3cf0306447c5bfbe879d51cb2fe9] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp, totalSize=33.8 K 2024-11-25T05:56:01,207 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 081d283367d742c69408594f1dadb720, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732514158492 2024-11-25T05:56:01,207 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 69b1926e93e84f66b587b0d48d51f50b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732514158689 2024-11-25T05:56:01,207 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 428e3cf0306447c5bfbe879d51cb2fe9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732514160663 2024-11-25T05:56:01,210 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0a5da8d44418b4369e0cb022c813f46/A of f0a5da8d44418b4369e0cb022c813f46 into 51da812ae12a4385b17cd9d8a5f15acf(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:56:01,210 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:56:01,210 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., storeName=f0a5da8d44418b4369e0cb022c813f46/A, priority=13, startTime=1732514160762; duration=0sec 2024-11-25T05:56:01,210 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:01,210 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0a5da8d44418b4369e0cb022c813f46:A 2024-11-25T05:56:01,215 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0a5da8d44418b4369e0cb022c813f46#C#compaction#265 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:56:01,215 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/f89472ffc181412c8729d3d2d29ace40 is 50, key is test_row_0/C:col10/1732514160663/Put/seqid=0 2024-11-25T05:56:01,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742133_1309 (size=12745) 2024-11-25T05:56:01,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/e06509eb75de4f30b3e640f0f7990f29 2024-11-25T05:56:01,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/2f37435177d84f89ac9047b08c8c614a as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/2f37435177d84f89ac9047b08c8c614a 2024-11-25T05:56:01,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/2f37435177d84f89ac9047b08c8c614a, entries=200, sequenceid=303, filesize=14.4 K 2024-11-25T05:56:01,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/37629070169240e6ba441ac5c3369861 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/37629070169240e6ba441ac5c3369861 2024-11-25T05:56:01,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/37629070169240e6ba441ac5c3369861, entries=150, sequenceid=303, filesize=12.0 K 2024-11-25T05:56:01,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/e06509eb75de4f30b3e640f0f7990f29 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/e06509eb75de4f30b3e640f0f7990f29 2024-11-25T05:56:01,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/e06509eb75de4f30b3e640f0f7990f29, entries=150, sequenceid=303, filesize=12.0 K 2024-11-25T05:56:01,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=134.18 KB/137400 for f0a5da8d44418b4369e0cb022c813f46 in 517ms, sequenceid=303, compaction requested=false 2024-11-25T05:56:01,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:56:01,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-25T05:56:01,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:01,286 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing f0a5da8d44418b4369e0cb022c813f46 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-25T05:56:01,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=A 2024-11-25T05:56:01,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:56:01,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=B 2024-11-25T05:56:01,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:56:01,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=C 2024-11-25T05:56:01,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:56:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-25T05:56:01,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/4647b2e34bc64cfb95bec2e76b6159b6 is 50, key is test_row_0/A:col10/1732514160869/Put/seqid=0 2024-11-25T05:56:01,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742134_1310 (size=12301) 2024-11-25T05:56:01,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f0a5da8d44418b4369e0cb022c813f46 2024-11-25T05:56:01,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:56:01,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:01,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514221511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:01,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:01,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:01,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:01,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:01,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53848 deadline: 1732514221512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:01,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:01,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:01,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:01,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:01,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514221513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:01,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:01,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:01,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:01,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:01,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514221516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:01,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:01,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:01,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:01,627 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/f89472ffc181412c8729d3d2d29ace40 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/f89472ffc181412c8729d3d2d29ace40 2024-11-25T05:56:01,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:01,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:01,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514221622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:01,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514221622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:01,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:01,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:01,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:01,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:01,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:01,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:01,632 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0a5da8d44418b4369e0cb022c813f46/C of f0a5da8d44418b4369e0cb022c813f46 into f89472ffc181412c8729d3d2d29ace40(size=12.4 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:56:01,632 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:56:01,632 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., storeName=f0a5da8d44418b4369e0cb022c813f46/C, priority=13, startTime=1732514160762; duration=0sec 2024-11-25T05:56:01,632 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:01,632 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0a5da8d44418b4369e0cb022c813f46:C 2024-11-25T05:56:01,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:01,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514221632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:01,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:01,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:01,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:01,715 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/4647b2e34bc64cfb95bec2e76b6159b6 2024-11-25T05:56:01,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/335a8b9df27f46f591b87f69e401c94b is 50, key is test_row_0/B:col10/1732514160869/Put/seqid=0 2024-11-25T05:56:01,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742135_1311 (size=12301) 2024-11-25T05:56:01,740 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/335a8b9df27f46f591b87f69e401c94b 2024-11-25T05:56:01,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/819d61e7efed40c2a4f1ef7f6683034a is 50, key is test_row_0/C:col10/1732514160869/Put/seqid=0 2024-11-25T05:56:01,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742136_1312 (size=12301) 2024-11-25T05:56:01,759 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/819d61e7efed40c2a4f1ef7f6683034a 2024-11-25T05:56:01,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/4647b2e34bc64cfb95bec2e76b6159b6 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/4647b2e34bc64cfb95bec2e76b6159b6 2024-11-25T05:56:01,774 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/4647b2e34bc64cfb95bec2e76b6159b6, entries=150, sequenceid=342, filesize=12.0 K 2024-11-25T05:56:01,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/335a8b9df27f46f591b87f69e401c94b as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/335a8b9df27f46f591b87f69e401c94b 2024-11-25T05:56:01,780 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/335a8b9df27f46f591b87f69e401c94b, entries=150, sequenceid=342, filesize=12.0 K 2024-11-25T05:56:01,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/819d61e7efed40c2a4f1ef7f6683034a as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/819d61e7efed40c2a4f1ef7f6683034a 2024-11-25T05:56:01,800 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/819d61e7efed40c2a4f1ef7f6683034a, entries=150, sequenceid=342, filesize=12.0 K 2024-11-25T05:56:01,802 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for f0a5da8d44418b4369e0cb022c813f46 in 516ms, sequenceid=342, compaction requested=true 2024-11-25T05:56:01,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:56:01,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:01,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-25T05:56:01,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-25T05:56:01,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-25T05:56:01,808 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-25T05:56:01,808 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1360 sec 2024-11-25T05:56:01,813 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.1460 sec 2024-11-25T05:56:01,835 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0a5da8d44418b4369e0cb022c813f46 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-25T05:56:01,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f0a5da8d44418b4369e0cb022c813f46 2024-11-25T05:56:01,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=A 2024-11-25T05:56:01,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:01,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=B 2024-11-25T05:56:01,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:01,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=C 2024-11-25T05:56:01,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:01,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/a03b67ba769248a69b2031e3c1abdc60 is 50, key is test_row_0/A:col10/1732514161833/Put/seqid=0 2024-11-25T05:56:01,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742137_1313 (size=17181) 2024-11-25T05:56:01,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:01,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514221888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:01,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732514161869, max timestamp=1732514161889Num uniques 859; 2024-11-25T05:56:01,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732514161869, max timestamp=1732514161889Num uniques 859; 2024-11-25T05:56:01,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T05:56:01,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732514161869, max timestamp=1732514161889Num uniques 859; 2024-11-25T05:56:01,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T05:56:01,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T05:56:01,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:01,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:01,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:01,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:01,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514221939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:01,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:01,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:01,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:01,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:01,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514221943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:01,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:01,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:01,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:02,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:02,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514222002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:02,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:02,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:02,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:02,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:02,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53848 deadline: 1732514222022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:02,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:02,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:02,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:02,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:02,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514222052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:02,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:02,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514222052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:02,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:02,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:02,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:02,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:02,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:02,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:02,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:02,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514222212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:02,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:02,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:02,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:02,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:02,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514222262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:02,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:02,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514222262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:02,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:02,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:02,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:02,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:02,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:02,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:02,277 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/a03b67ba769248a69b2031e3c1abdc60 2024-11-25T05:56:02,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/183433453ef4468b966de4f98b678708 is 50, key is test_row_0/B:col10/1732514161833/Put/seqid=0 2024-11-25T05:56:02,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742138_1314 (size=12301) 2024-11-25T05:56:02,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:02,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514222522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:02,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:02,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:02,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:02,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:02,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514222572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:02,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:02,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:02,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:02,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514222572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:02,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:02,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:02,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:02,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:02,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/183433453ef4468b966de4f98b678708 2024-11-25T05:56:02,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/7d8077c019f7477fa7c2b74369109638 is 50, key is test_row_0/C:col10/1732514161833/Put/seqid=0 2024-11-25T05:56:02,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742139_1315 (size=12301) 2024-11-25T05:56:02,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-25T05:56:02,822 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:56:02,823 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:56:02,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-25T05:56:02,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-25T05:56:02,824 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:56:02,825 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:56:02,825 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:56:02,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-25T05:56:02,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-25T05:56:02,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:02,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:56:02,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:02,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:02,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:02,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:03,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:03,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53848 deadline: 1732514223032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:03,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:03,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514223032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:03,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:03,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:03,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:03,036 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_0' in f0a5da8d44418b4369e0cb022c813f46 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:56:03,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:03,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:03,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:03,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:03,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514223082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:03,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:03,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:03,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:03,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:03,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514223082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:03,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:03,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:03,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:03,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/7d8077c019f7477fa7c2b74369109638 2024-11-25T05:56:03,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/a03b67ba769248a69b2031e3c1abdc60 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/a03b67ba769248a69b2031e3c1abdc60 2024-11-25T05:56:03,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/a03b67ba769248a69b2031e3c1abdc60, entries=250, sequenceid=355, filesize=16.8 K 2024-11-25T05:56:03,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/183433453ef4468b966de4f98b678708 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/183433453ef4468b966de4f98b678708 2024-11-25T05:56:03,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/183433453ef4468b966de4f98b678708, entries=150, sequenceid=355, filesize=12.0 K 2024-11-25T05:56:03,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/7d8077c019f7477fa7c2b74369109638 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/7d8077c019f7477fa7c2b74369109638 2024-11-25T05:56:03,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/7d8077c019f7477fa7c2b74369109638, entries=150, sequenceid=355, filesize=12.0 K 2024-11-25T05:56:03,115 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for f0a5da8d44418b4369e0cb022c813f46 in 1280ms, sequenceid=355, compaction requested=true 2024-11-25T05:56:03,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:56:03,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0a5da8d44418b4369e0cb022c813f46:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:56:03,116 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T05:56:03,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:03,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0a5da8d44418b4369e0cb022c813f46:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:56:03,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:03,116 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T05:56:03,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0a5da8d44418b4369e0cb022c813f46:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:56:03,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:56:03,117 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49648 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T05:56:03,117 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56968 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T05:56:03,117 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): f0a5da8d44418b4369e0cb022c813f46/B is initiating minor compaction (all files) 2024-11-25T05:56:03,117 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): f0a5da8d44418b4369e0cb022c813f46/A is initiating minor compaction (all files) 2024-11-25T05:56:03,117 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0a5da8d44418b4369e0cb022c813f46/A in TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:03,117 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0a5da8d44418b4369e0cb022c813f46/B in TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:03,117 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/c12eb53113144272bc6373ac7d0680ed, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/37629070169240e6ba441ac5c3369861, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/335a8b9df27f46f591b87f69e401c94b, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/183433453ef4468b966de4f98b678708] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp, totalSize=48.5 K 2024-11-25T05:56:03,117 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/51da812ae12a4385b17cd9d8a5f15acf, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/2f37435177d84f89ac9047b08c8c614a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/4647b2e34bc64cfb95bec2e76b6159b6, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/a03b67ba769248a69b2031e3c1abdc60] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp, totalSize=55.6 K 2024-11-25T05:56:03,117 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting c12eb53113144272bc6373ac7d0680ed, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732514160663 2024-11-25T05:56:03,117 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51da812ae12a4385b17cd9d8a5f15acf, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732514160663 2024-11-25T05:56:03,118 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 37629070169240e6ba441ac5c3369861, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732514160758 2024-11-25T05:56:03,118 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f37435177d84f89ac9047b08c8c614a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732514160746 2024-11-25T05:56:03,118 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 335a8b9df27f46f591b87f69e401c94b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732514160860 2024-11-25T05:56:03,118 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4647b2e34bc64cfb95bec2e76b6159b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732514160860 2024-11-25T05:56:03,118 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 183433453ef4468b966de4f98b678708, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732514161515 2024-11-25T05:56:03,118 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting a03b67ba769248a69b2031e3c1abdc60, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732514161508 2024-11-25T05:56:03,125 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0a5da8d44418b4369e0cb022c813f46#B#compaction#272 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:56:03,126 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/3b2b015937cf484eb7986e8f4563acf7 is 50, key is test_row_0/B:col10/1732514161833/Put/seqid=0 2024-11-25T05:56:03,126 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0a5da8d44418b4369e0cb022c813f46#A#compaction#273 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:56:03,127 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/fa6c8dc1ae22495ca1ee2245bdc3961f is 50, key is test_row_0/A:col10/1732514161833/Put/seqid=0 2024-11-25T05:56:03,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-25T05:56:03,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:03,130 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing f0a5da8d44418b4369e0cb022c813f46 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-11-25T05:56:03,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=A 2024-11-25T05:56:03,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:56:03,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=B 2024-11-25T05:56:03,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:56:03,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=C 2024-11-25T05:56:03,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:56:03,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-25T05:56:03,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/910ea26dca8e480bbee9029d29ee6ef3 is 50, key is test_row_0/A:col10/1732514161937/Put/seqid=0 2024-11-25T05:56:03,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742140_1316 (size=12881) 2024-11-25T05:56:03,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742141_1317 (size=12881) 2024-11-25T05:56:03,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742142_1318 (size=12301) 2024-11-25T05:56:03,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-25T05:56:03,549 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/3b2b015937cf484eb7986e8f4563acf7 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/3b2b015937cf484eb7986e8f4563acf7 2024-11-25T05:56:03,554 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0a5da8d44418b4369e0cb022c813f46/B of f0a5da8d44418b4369e0cb022c813f46 into 3b2b015937cf484eb7986e8f4563acf7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:56:03,554 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:56:03,554 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., storeName=f0a5da8d44418b4369e0cb022c813f46/B, priority=12, startTime=1732514163116; duration=0sec 2024-11-25T05:56:03,554 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:56:03,554 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0a5da8d44418b4369e0cb022c813f46:B 2024-11-25T05:56:03,554 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T05:56:03,555 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/910ea26dca8e480bbee9029d29ee6ef3 2024-11-25T05:56:03,555 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49648 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T05:56:03,555 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): f0a5da8d44418b4369e0cb022c813f46/C is initiating minor compaction (all files) 2024-11-25T05:56:03,555 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0a5da8d44418b4369e0cb022c813f46/C in TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:03,555 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/f89472ffc181412c8729d3d2d29ace40, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/e06509eb75de4f30b3e640f0f7990f29, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/819d61e7efed40c2a4f1ef7f6683034a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/7d8077c019f7477fa7c2b74369109638] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp, totalSize=48.5 K 2024-11-25T05:56:03,556 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting f89472ffc181412c8729d3d2d29ace40, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732514160663 2024-11-25T05:56:03,557 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting e06509eb75de4f30b3e640f0f7990f29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1732514160758 2024-11-25T05:56:03,558 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/fa6c8dc1ae22495ca1ee2245bdc3961f as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/fa6c8dc1ae22495ca1ee2245bdc3961f 2024-11-25T05:56:03,558 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 819d61e7efed40c2a4f1ef7f6683034a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732514160860 2024-11-25T05:56:03,558 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d8077c019f7477fa7c2b74369109638, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732514161515 2024-11-25T05:56:03,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/f75ccca4b45c41e3858ca3ea631b8b6f is 50, key is test_row_0/B:col10/1732514161937/Put/seqid=0 2024-11-25T05:56:03,568 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0a5da8d44418b4369e0cb022c813f46/A of f0a5da8d44418b4369e0cb022c813f46 into fa6c8dc1ae22495ca1ee2245bdc3961f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:56:03,568 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:56:03,568 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., storeName=f0a5da8d44418b4369e0cb022c813f46/A, priority=12, startTime=1732514163116; duration=0sec 2024-11-25T05:56:03,568 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:03,568 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0a5da8d44418b4369e0cb022c813f46:A 2024-11-25T05:56:03,573 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0a5da8d44418b4369e0cb022c813f46#C#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:56:03,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742143_1319 (size=12301) 2024-11-25T05:56:03,574 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/3ab1043331c2462b9d2af6b84eaebd6b is 50, key is test_row_0/C:col10/1732514161833/Put/seqid=0 2024-11-25T05:56:03,574 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/f75ccca4b45c41e3858ca3ea631b8b6f 2024-11-25T05:56:03,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/193596e43f4147e9b8f4dd6ce204e891 is 50, key is test_row_0/C:col10/1732514161937/Put/seqid=0 2024-11-25T05:56:03,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742144_1320 (size=12881) 2024-11-25T05:56:03,630 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/3ab1043331c2462b9d2af6b84eaebd6b as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/3ab1043331c2462b9d2af6b84eaebd6b 2024-11-25T05:56:03,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742145_1321 (size=12301) 2024-11-25T05:56:03,637 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0a5da8d44418b4369e0cb022c813f46/C of f0a5da8d44418b4369e0cb022c813f46 into 3ab1043331c2462b9d2af6b84eaebd6b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:56:03,637 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:56:03,637 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., storeName=f0a5da8d44418b4369e0cb022c813f46/C, priority=12, startTime=1732514163116; duration=0sec 2024-11-25T05:56:03,637 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:03,637 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0a5da8d44418b4369e0cb022c813f46:C 2024-11-25T05:56:03,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-25T05:56:04,035 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/193596e43f4147e9b8f4dd6ce204e891 2024-11-25T05:56:04,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/910ea26dca8e480bbee9029d29ee6ef3 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/910ea26dca8e480bbee9029d29ee6ef3 2024-11-25T05:56:04,041 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/910ea26dca8e480bbee9029d29ee6ef3, entries=150, sequenceid=389, filesize=12.0 K 2024-11-25T05:56:04,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/f75ccca4b45c41e3858ca3ea631b8b6f as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/f75ccca4b45c41e3858ca3ea631b8b6f 2024-11-25T05:56:04,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f0a5da8d44418b4369e0cb022c813f46 2024-11-25T05:56:04,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:56:04,046 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/f75ccca4b45c41e3858ca3ea631b8b6f, entries=150, sequenceid=389, filesize=12.0 K 2024-11-25T05:56:04,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/193596e43f4147e9b8f4dd6ce204e891 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/193596e43f4147e9b8f4dd6ce204e891 2024-11-25T05:56:04,050 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/193596e43f4147e9b8f4dd6ce204e891, entries=150, sequenceid=389, filesize=12.0 K 2024-11-25T05:56:04,051 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=13.42 KB/13740 for f0a5da8d44418b4369e0cb022c813f46 in 921ms, sequenceid=389, compaction requested=false 2024-11-25T05:56:04,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:56:04,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:04,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-25T05:56:04,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-25T05:56:04,054 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-25T05:56:04,054 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2270 sec 2024-11-25T05:56:04,056 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.2310 sec 2024-11-25T05:56:04,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f0a5da8d44418b4369e0cb022c813f46 2024-11-25T05:56:04,060 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0a5da8d44418b4369e0cb022c813f46 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-25T05:56:04,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=A 2024-11-25T05:56:04,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:04,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=B 2024-11-25T05:56:04,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:04,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=C 2024-11-25T05:56:04,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:04,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/36292f53b08f4e96b288b546f6696ef8 is 50, key is test_row_0/A:col10/1732514164060/Put/seqid=0 2024-11-25T05:56:04,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742146_1322 (size=12301) 2024-11-25T05:56:04,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732514164108, max timestamp=1732514164134Num uniques 859; 2024-11-25T05:56:04,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732514164108, max timestamp=1732514164134Num uniques 859; 2024-11-25T05:56:04,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732514164108, max timestamp=1732514164134Num uniques 859; 2024-11-25T05:56:04,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T05:56:04,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T05:56:04,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T05:56:04,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:04,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514224184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:04,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:04,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:04,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:04,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:04,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514224192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:04,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:04,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:04,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:04,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:04,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514224194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:04,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:04,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:04,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:04,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:04,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514224302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:04,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:04,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:04,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:04,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:04,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514224302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:04,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:04,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:04,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:04,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:04,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514224302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:04,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:04,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:04,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:04,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/36292f53b08f4e96b288b546f6696ef8 2024-11-25T05:56:04,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/94d5a166f98a4f009afde0b3f0117c6f is 50, key is test_row_0/B:col10/1732514164060/Put/seqid=0 2024-11-25T05:56:04,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742147_1323 (size=12301) 2024-11-25T05:56:04,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:04,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514224511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:04,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:04,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:04,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:04,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:04,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514224512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:04,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:04,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:04,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:04,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:04,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514224512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:04,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:04,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:04,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:04,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:04,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514224822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:04,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:04,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:04,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:04,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:04,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514224822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:04,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:04,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:04,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:04,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:04,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514224822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:04,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:04,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:04,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:04,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/94d5a166f98a4f009afde0b3f0117c6f 2024-11-25T05:56:04,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/2393570b5f764980b6e7353c40829802 is 50, key is test_row_0/C:col10/1732514164060/Put/seqid=0 2024-11-25T05:56:04,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742148_1324 (size=12301) 2024-11-25T05:56:04,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-25T05:56:04,972 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:56:04,972 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:56:04,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-25T05:56:04,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-25T05:56:04,974 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:56:04,974 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:56:04,974 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:56:05,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:05,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53848 deadline: 1732514225042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:05,045 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_0' in f0a5da8d44418b4369e0cb022c813f46 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:56:05,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:05,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:05,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:05,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-25T05:56:05,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-25T05:56:05,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:05,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:56:05,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:05,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:05,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:05,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:05,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-25T05:56:05,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:05,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:56:05,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:05,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:05,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:05,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:05,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-25T05:56:05,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/2393570b5f764980b6e7353c40829802 2024-11-25T05:56:05,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/36292f53b08f4e96b288b546f6696ef8 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/36292f53b08f4e96b288b546f6696ef8 2024-11-25T05:56:05,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/36292f53b08f4e96b288b546f6696ef8, entries=150, sequenceid=402, filesize=12.0 K 2024-11-25T05:56:05,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/94d5a166f98a4f009afde0b3f0117c6f as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/94d5a166f98a4f009afde0b3f0117c6f 2024-11-25T05:56:05,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/94d5a166f98a4f009afde0b3f0117c6f, entries=150, sequenceid=402, filesize=12.0 K 2024-11-25T05:56:05,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/2393570b5f764980b6e7353c40829802 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/2393570b5f764980b6e7353c40829802 2024-11-25T05:56:05,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/2393570b5f764980b6e7353c40829802, entries=150, sequenceid=402, filesize=12.0 K 2024-11-25T05:56:05,312 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f0a5da8d44418b4369e0cb022c813f46 in 1252ms, sequenceid=402, compaction requested=true 2024-11-25T05:56:05,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:56:05,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0a5da8d44418b4369e0cb022c813f46:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:56:05,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:05,313 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:56:05,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0a5da8d44418b4369e0cb022c813f46:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:56:05,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:05,313 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:56:05,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0a5da8d44418b4369e0cb022c813f46:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:56:05,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:56:05,314 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:56:05,314 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): f0a5da8d44418b4369e0cb022c813f46/B is initiating minor compaction (all files) 2024-11-25T05:56:05,314 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0a5da8d44418b4369e0cb022c813f46/B in TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:05,314 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/3b2b015937cf484eb7986e8f4563acf7, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/f75ccca4b45c41e3858ca3ea631b8b6f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/94d5a166f98a4f009afde0b3f0117c6f] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp, totalSize=36.6 K 2024-11-25T05:56:05,314 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:56:05,314 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): f0a5da8d44418b4369e0cb022c813f46/A is initiating minor compaction (all files) 2024-11-25T05:56:05,314 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0a5da8d44418b4369e0cb022c813f46/A in TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:05,314 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b2b015937cf484eb7986e8f4563acf7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732514161515 2024-11-25T05:56:05,314 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/fa6c8dc1ae22495ca1ee2245bdc3961f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/910ea26dca8e480bbee9029d29ee6ef3, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/36292f53b08f4e96b288b546f6696ef8] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp, totalSize=36.6 K 2024-11-25T05:56:05,315 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting f75ccca4b45c41e3858ca3ea631b8b6f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1732514161933 2024-11-25T05:56:05,315 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa6c8dc1ae22495ca1ee2245bdc3961f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732514161515 2024-11-25T05:56:05,315 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 94d5a166f98a4f009afde0b3f0117c6f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732514164048 2024-11-25T05:56:05,315 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 910ea26dca8e480bbee9029d29ee6ef3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1732514161933 2024-11-25T05:56:05,315 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36292f53b08f4e96b288b546f6696ef8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732514164048 2024-11-25T05:56:05,322 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0a5da8d44418b4369e0cb022c813f46#A#compaction#281 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:56:05,322 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/5d267b80fde3477799b0398c2cd1593f is 50, key is test_row_0/A:col10/1732514164060/Put/seqid=0 2024-11-25T05:56:05,323 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0a5da8d44418b4369e0cb022c813f46#B#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:56:05,324 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/ccb4e5e4ab0840cf93a393f757a1f48f is 50, key is test_row_0/B:col10/1732514164060/Put/seqid=0 2024-11-25T05:56:05,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742150_1326 (size=12983) 2024-11-25T05:56:05,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742149_1325 (size=12983) 2024-11-25T05:56:05,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f0a5da8d44418b4369e0cb022c813f46 2024-11-25T05:56:05,337 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0a5da8d44418b4369e0cb022c813f46 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-25T05:56:05,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=A 2024-11-25T05:56:05,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-25T05:56:05,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=B 2024-11-25T05:56:05,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-25T05:56:05,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=C 2024-11-25T05:56:05,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-25T05:56:05,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/9f4a262a0cb44a7ab2dc3fc6d0cd0984 is 50, key is test_row_0/A:col10/1732514165336/Put/seqid=0 2024-11-25T05:56:05,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742151_1327 (size=14741) 2024-11-25T05:56:05,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:05,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514225347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:05,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:05,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:05,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:05,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514225347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:05,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:05,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:05,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:05,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:05,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:05,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514225354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:05,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:05,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:05,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:05,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-25T05:56:05,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:05,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:56:05,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:05,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:05,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:05,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:05,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:05,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514225462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:05,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:05,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:05,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514225462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:05,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:05,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:05,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:05,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:05,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:05,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514225462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:05,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:05,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:05,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:05,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:05,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-25T05:56:05,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:05,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:56:05,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:05,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:05,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:05,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:05,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-25T05:56:05,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:05,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514225672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514225672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:05,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:05,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:05,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:05,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:05,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514225672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:05,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:05,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:05,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:05,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:05,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:05,732 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/ccb4e5e4ab0840cf93a393f757a1f48f as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/ccb4e5e4ab0840cf93a393f757a1f48f 2024-11-25T05:56:05,732 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/5d267b80fde3477799b0398c2cd1593f as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/5d267b80fde3477799b0398c2cd1593f 2024-11-25T05:56:05,737 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0a5da8d44418b4369e0cb022c813f46/B of f0a5da8d44418b4369e0cb022c813f46 into ccb4e5e4ab0840cf93a393f757a1f48f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:56:05,737 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0a5da8d44418b4369e0cb022c813f46/A of f0a5da8d44418b4369e0cb022c813f46 into 5d267b80fde3477799b0398c2cd1593f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:56:05,737 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:56:05,737 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:56:05,737 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., storeName=f0a5da8d44418b4369e0cb022c813f46/B, priority=13, startTime=1732514165313; duration=0sec 2024-11-25T05:56:05,737 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., storeName=f0a5da8d44418b4369e0cb022c813f46/A, priority=13, startTime=1732514165313; duration=0sec 2024-11-25T05:56:05,737 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:56:05,737 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:56:05,737 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0a5da8d44418b4369e0cb022c813f46:A 2024-11-25T05:56:05,737 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0a5da8d44418b4369e0cb022c813f46:B 2024-11-25T05:56:05,737 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:56:05,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-25T05:56:05,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:05,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:56:05,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:05,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:05,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:05,739 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:56:05,739 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): f0a5da8d44418b4369e0cb022c813f46/C is initiating minor compaction (all files) 2024-11-25T05:56:05,739 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0a5da8d44418b4369e0cb022c813f46/C in TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:05,739 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/3ab1043331c2462b9d2af6b84eaebd6b, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/193596e43f4147e9b8f4dd6ce204e891, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/2393570b5f764980b6e7353c40829802] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp, totalSize=36.6 K 2024-11-25T05:56:05,739 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ab1043331c2462b9d2af6b84eaebd6b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732514161515 2024-11-25T05:56:05,740 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 193596e43f4147e9b8f4dd6ce204e891, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1732514161933 2024-11-25T05:56:05,740 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2393570b5f764980b6e7353c40829802, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732514164048 2024-11-25T05:56:05,746 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0a5da8d44418b4369e0cb022c813f46#C#compaction#284 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:56:05,746 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/e01c5df790c0441098427d6cd15d4a81 is 50, key is test_row_0/C:col10/1732514164060/Put/seqid=0 2024-11-25T05:56:05,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/9f4a262a0cb44a7ab2dc3fc6d0cd0984 2024-11-25T05:56:05,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742152_1328 (size=12983) 2024-11-25T05:56:05,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/73d2d2a810ca4cdfb5a09a70e90ae1f0 is 50, key is test_row_0/B:col10/1732514165336/Put/seqid=0 2024-11-25T05:56:05,757 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/e01c5df790c0441098427d6cd15d4a81 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/e01c5df790c0441098427d6cd15d4a81 2024-11-25T05:56:05,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742153_1329 (size=12301) 2024-11-25T05:56:05,763 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0a5da8d44418b4369e0cb022c813f46/C of f0a5da8d44418b4369e0cb022c813f46 into e01c5df790c0441098427d6cd15d4a81(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:56:05,763 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:56:05,763 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., storeName=f0a5da8d44418b4369e0cb022c813f46/C, priority=13, startTime=1732514165313; duration=0sec 2024-11-25T05:56:05,763 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:05,763 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0a5da8d44418b4369e0cb022c813f46:C 2024-11-25T05:56:05,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-25T05:56:05,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:05,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:56:05,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:05,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:05,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:05,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:05,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514225982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:05,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:05,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:05,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:05,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514225982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:05,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:05,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:05,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:05,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514225992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:05,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:05,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:05,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:06,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-25T05:56:06,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:06,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:56:06,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:06,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:06,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-25T05:56:06,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/73d2d2a810ca4cdfb5a09a70e90ae1f0 2024-11-25T05:56:06,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/dd3fcb5195db4a168a65460b4861b9cb is 50, key is test_row_0/C:col10/1732514165336/Put/seqid=0 2024-11-25T05:56:06,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742154_1330 (size=12301) 2024-11-25T05:56:06,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-25T05:56:06,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:06,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:56:06,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:06,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:06,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:06,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-25T05:56:06,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:06,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:56:06,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:06,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:06,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:06,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:06,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514226492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:06,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:06,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:06,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:06,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514226492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:06,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:06,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:06,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:06,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:06,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:06,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514226502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:06,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:06,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:06,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:06,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-25T05:56:06,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:06,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:56:06,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:06,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:06,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:06,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:06,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/dd3fcb5195db4a168a65460b4861b9cb 2024-11-25T05:56:06,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/9f4a262a0cb44a7ab2dc3fc6d0cd0984 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/9f4a262a0cb44a7ab2dc3fc6d0cd0984 2024-11-25T05:56:06,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/9f4a262a0cb44a7ab2dc3fc6d0cd0984, entries=200, sequenceid=440, filesize=14.4 K 2024-11-25T05:56:06,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/73d2d2a810ca4cdfb5a09a70e90ae1f0 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/73d2d2a810ca4cdfb5a09a70e90ae1f0 2024-11-25T05:56:06,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/73d2d2a810ca4cdfb5a09a70e90ae1f0, entries=150, sequenceid=440, filesize=12.0 K 2024-11-25T05:56:06,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/dd3fcb5195db4a168a65460b4861b9cb as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/dd3fcb5195db4a168a65460b4861b9cb 2024-11-25T05:56:06,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/dd3fcb5195db4a168a65460b4861b9cb, entries=150, sequenceid=440, filesize=12.0 K 2024-11-25T05:56:06,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for f0a5da8d44418b4369e0cb022c813f46 in 1254ms, sequenceid=440, compaction requested=false 2024-11-25T05:56:06,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:56:06,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-25T05:56:06,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:06,663 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing f0a5da8d44418b4369e0cb022c813f46 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-25T05:56:06,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=A 2024-11-25T05:56:06,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:06,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=B 2024-11-25T05:56:06,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:06,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=C 2024-11-25T05:56:06,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:06,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/135ccc15fd334f8193eb46fa7e9281ad is 50, key is test_row_0/A:col10/1732514165338/Put/seqid=0 2024-11-25T05:56:06,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742155_1331 (size=12301) 2024-11-25T05:56:07,072 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/135ccc15fd334f8193eb46fa7e9281ad 2024-11-25T05:56:07,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/d15d099b2b0d4ae29f791cc2dc8e8652 is 50, key is test_row_0/B:col10/1732514165338/Put/seqid=0 2024-11-25T05:56:07,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742156_1332 (size=12301) 2024-11-25T05:56:07,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-25T05:56:07,491 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/d15d099b2b0d4ae29f791cc2dc8e8652 2024-11-25T05:56:07,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/8bce6284ccd84207b662bf4208e53336 is 50, key is test_row_0/C:col10/1732514165338/Put/seqid=0 2024-11-25T05:56:07,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742157_1333 (size=12301) 2024-11-25T05:56:07,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:56:07,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f0a5da8d44418b4369e0cb022c813f46 2024-11-25T05:56:07,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732514167537, max timestamp=1732514167549Num uniques 859; 2024-11-25T05:56:07,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T05:56:07,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732514167537, max timestamp=1732514167549Num uniques 859; 2024-11-25T05:56:07,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T05:56:07,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732514167537, max timestamp=1732514167549Num uniques 859; 2024-11-25T05:56:07,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T05:56:07,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732514167574, max timestamp=1732514167586Num uniques 1018; 2024-11-25T05:56:07,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732514167574, max timestamp=1732514167586Num uniques 1018; 2024-11-25T05:56:07,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-25T05:56:07,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732514167574, max timestamp=1732514167586Num uniques 1018; 2024-11-25T05:56:07,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-25T05:56:07,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-25T05:56:07,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:07,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514227614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:07,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:07,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:07,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:07,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:07,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514227615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:07,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:07,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:07,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:07,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:07,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514227616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:07,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:07,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:07,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:07,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:07,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514227722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:07,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:07,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:07,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:07,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514227722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:07,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:07,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:07,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:07,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:07,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:07,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514227732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:07,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:07,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:07,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:07,903 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/8bce6284ccd84207b662bf4208e53336 2024-11-25T05:56:07,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/135ccc15fd334f8193eb46fa7e9281ad as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/135ccc15fd334f8193eb46fa7e9281ad 2024-11-25T05:56:07,910 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/135ccc15fd334f8193eb46fa7e9281ad, entries=150, sequenceid=452, filesize=12.0 K 2024-11-25T05:56:07,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/d15d099b2b0d4ae29f791cc2dc8e8652 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/d15d099b2b0d4ae29f791cc2dc8e8652 2024-11-25T05:56:07,914 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/d15d099b2b0d4ae29f791cc2dc8e8652, entries=150, sequenceid=452, filesize=12.0 K 2024-11-25T05:56:07,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/8bce6284ccd84207b662bf4208e53336 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/8bce6284ccd84207b662bf4208e53336 2024-11-25T05:56:07,919 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/8bce6284ccd84207b662bf4208e53336, entries=150, sequenceid=452, filesize=12.0 K 2024-11-25T05:56:07,920 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for f0a5da8d44418b4369e0cb022c813f46 in 1257ms, sequenceid=452, compaction requested=true 2024-11-25T05:56:07,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:56:07,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:07,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-25T05:56:07,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-25T05:56:07,923 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-25T05:56:07,923 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9480 sec 2024-11-25T05:56:07,925 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.9520 sec 2024-11-25T05:56:07,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f0a5da8d44418b4369e0cb022c813f46 2024-11-25T05:56:07,935 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0a5da8d44418b4369e0cb022c813f46 3/3 column families, dataSize=147.60 KB heapSize=414.61 KB 2024-11-25T05:56:07,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=A 2024-11-25T05:56:07,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:56:07,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=B 2024-11-25T05:56:07,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:56:07,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=C 2024-11-25T05:56:07,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:56:07,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/2b1a802aef56451ca2ae84e5e46fb34b is 50, key is test_row_0/A:col10/1732514167613/Put/seqid=0 2024-11-25T05:56:07,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742158_1334 (size=14741) 2024-11-25T05:56:07,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:07,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514227947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:07,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:07,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:07,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:07,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:07,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514227947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:07,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:07,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:07,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:07,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514227948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:07,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:07,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:07,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:07,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:08,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:08,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514228062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:08,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:08,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:08,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:08,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:08,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514228062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:08,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:08,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:08,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:08,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:08,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514228062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:08,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:08,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:08,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:08,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:08,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514228272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:08,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:08,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:08,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514228272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:08,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:08,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:08,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:08,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:08,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:08,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514228272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:08,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:08,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:08,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:08,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:08,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/2b1a802aef56451ca2ae84e5e46fb34b 2024-11-25T05:56:08,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/e195926933064cc28bdebfee6984cc11 is 50, key is test_row_0/B:col10/1732514167613/Put/seqid=0 2024-11-25T05:56:08,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742159_1335 (size=12301) 2024-11-25T05:56:08,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:08,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514228582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:08,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:08,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:08,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:08,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:08,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514228582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:08,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:08,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:08,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:08,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:08,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514228582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:08,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:08,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:08,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:08,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/e195926933064cc28bdebfee6984cc11 2024-11-25T05:56:08,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/550ab89aa5cd474485c8cef3401e2bf8 is 50, key is test_row_0/C:col10/1732514167613/Put/seqid=0 2024-11-25T05:56:08,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742160_1336 (size=12301) 2024-11-25T05:56:09,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:09,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53848 deadline: 1732514229052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:09,057 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_0' in f0a5da8d44418b4369e0cb022c813f46 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:56:09,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:09,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:09,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:09,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:09,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53826 deadline: 1732514229082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:09,087 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_0' in f0a5da8d44418b4369e0cb022c813f46 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18246 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:56:09,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:09,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:09,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:09,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:09,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514229092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:09,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:09,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:09,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514229092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:09,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:09,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:09,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:09,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:09,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:09,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:09,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514229092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:09,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:09,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:09,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:09,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-25T05:56:09,133 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:56:09,133 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:56:09,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-25T05:56:09,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-25T05:56:09,134 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:56:09,135 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:56:09,135 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:56:09,164 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/550ab89aa5cd474485c8cef3401e2bf8 2024-11-25T05:56:09,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/2b1a802aef56451ca2ae84e5e46fb34b as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/2b1a802aef56451ca2ae84e5e46fb34b 2024-11-25T05:56:09,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/2b1a802aef56451ca2ae84e5e46fb34b, entries=200, sequenceid=506, filesize=14.4 K 2024-11-25T05:56:09,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/e195926933064cc28bdebfee6984cc11 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/e195926933064cc28bdebfee6984cc11 2024-11-25T05:56:09,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/e195926933064cc28bdebfee6984cc11, entries=150, sequenceid=506, filesize=12.0 K 2024-11-25T05:56:09,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/550ab89aa5cd474485c8cef3401e2bf8 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/550ab89aa5cd474485c8cef3401e2bf8 2024-11-25T05:56:09,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/550ab89aa5cd474485c8cef3401e2bf8, entries=150, sequenceid=506, filesize=12.0 K 2024-11-25T05:56:09,184 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~414.56 KB/424512, currentSize=40.25 KB/41220 for f0a5da8d44418b4369e0cb022c813f46 in 1249ms, sequenceid=506, compaction requested=true 2024-11-25T05:56:09,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:56:09,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0a5da8d44418b4369e0cb022c813f46:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:56:09,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:09,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0a5da8d44418b4369e0cb022c813f46:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:56:09,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:09,184 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T05:56:09,184 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T05:56:09,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0a5da8d44418b4369e0cb022c813f46:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:56:09,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:56:09,185 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T05:56:09,185 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): f0a5da8d44418b4369e0cb022c813f46/B is initiating minor compaction (all files) 2024-11-25T05:56:09,185 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0a5da8d44418b4369e0cb022c813f46/B in TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:09,186 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/ccb4e5e4ab0840cf93a393f757a1f48f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/73d2d2a810ca4cdfb5a09a70e90ae1f0, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/d15d099b2b0d4ae29f791cc2dc8e8652, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/e195926933064cc28bdebfee6984cc11] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp, totalSize=48.7 K 2024-11-25T05:56:09,186 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting ccb4e5e4ab0840cf93a393f757a1f48f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732514164048 2024-11-25T05:56:09,186 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54766 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T05:56:09,186 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 73d2d2a810ca4cdfb5a09a70e90ae1f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1732514164160 2024-11-25T05:56:09,186 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): f0a5da8d44418b4369e0cb022c813f46/A is initiating minor compaction (all files) 2024-11-25T05:56:09,186 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0a5da8d44418b4369e0cb022c813f46/A in TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:09,187 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/5d267b80fde3477799b0398c2cd1593f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/9f4a262a0cb44a7ab2dc3fc6d0cd0984, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/135ccc15fd334f8193eb46fa7e9281ad, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/2b1a802aef56451ca2ae84e5e46fb34b] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp, totalSize=53.5 K 2024-11-25T05:56:09,187 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting d15d099b2b0d4ae29f791cc2dc8e8652, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1732514165338 2024-11-25T05:56:09,187 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d267b80fde3477799b0398c2cd1593f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732514164048 2024-11-25T05:56:09,187 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting e195926933064cc28bdebfee6984cc11, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1732514167613 2024-11-25T05:56:09,187 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f4a262a0cb44a7ab2dc3fc6d0cd0984, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1732514164149 2024-11-25T05:56:09,188 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 135ccc15fd334f8193eb46fa7e9281ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1732514165338 2024-11-25T05:56:09,188 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b1a802aef56451ca2ae84e5e46fb34b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1732514167607 2024-11-25T05:56:09,200 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0a5da8d44418b4369e0cb022c813f46#A#compaction#293 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:56:09,200 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/988064dbdb2140d096bcfc20053fc029 is 50, key is test_row_0/A:col10/1732514167613/Put/seqid=0 2024-11-25T05:56:09,207 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0a5da8d44418b4369e0cb022c813f46#B#compaction#294 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:56:09,207 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/673395c53ff84f54a5fb3898e7572870 is 50, key is test_row_0/B:col10/1732514167613/Put/seqid=0 2024-11-25T05:56:09,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742162_1338 (size=13119) 2024-11-25T05:56:09,218 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/673395c53ff84f54a5fb3898e7572870 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/673395c53ff84f54a5fb3898e7572870 2024-11-25T05:56:09,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742161_1337 (size=13119) 2024-11-25T05:56:09,225 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0a5da8d44418b4369e0cb022c813f46/B of f0a5da8d44418b4369e0cb022c813f46 into 673395c53ff84f54a5fb3898e7572870(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:56:09,226 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:56:09,226 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., storeName=f0a5da8d44418b4369e0cb022c813f46/B, priority=12, startTime=1732514169184; duration=0sec 2024-11-25T05:56:09,226 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:56:09,226 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0a5da8d44418b4369e0cb022c813f46:B 2024-11-25T05:56:09,226 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T05:56:09,228 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T05:56:09,228 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): f0a5da8d44418b4369e0cb022c813f46/C is initiating minor compaction (all files) 2024-11-25T05:56:09,228 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0a5da8d44418b4369e0cb022c813f46/C in TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:09,228 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/e01c5df790c0441098427d6cd15d4a81, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/dd3fcb5195db4a168a65460b4861b9cb, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/8bce6284ccd84207b662bf4208e53336, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/550ab89aa5cd474485c8cef3401e2bf8] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp, totalSize=48.7 K 2024-11-25T05:56:09,228 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting e01c5df790c0441098427d6cd15d4a81, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732514164048 2024-11-25T05:56:09,228 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting dd3fcb5195db4a168a65460b4861b9cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1732514164160 2024-11-25T05:56:09,229 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/988064dbdb2140d096bcfc20053fc029 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/988064dbdb2140d096bcfc20053fc029 2024-11-25T05:56:09,229 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bce6284ccd84207b662bf4208e53336, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1732514165338 2024-11-25T05:56:09,229 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 550ab89aa5cd474485c8cef3401e2bf8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1732514167613 2024-11-25T05:56:09,237 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0a5da8d44418b4369e0cb022c813f46/A of f0a5da8d44418b4369e0cb022c813f46 into 988064dbdb2140d096bcfc20053fc029(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:56:09,237 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:56:09,237 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., storeName=f0a5da8d44418b4369e0cb022c813f46/A, priority=12, startTime=1732514169184; duration=0sec 2024-11-25T05:56:09,237 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:09,237 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0a5da8d44418b4369e0cb022c813f46:A 2024-11-25T05:56:09,240 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0a5da8d44418b4369e0cb022c813f46#C#compaction#295 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:56:09,240 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/e802cee469d44a22909ee199d82355ca is 50, key is test_row_0/C:col10/1732514167613/Put/seqid=0 2024-11-25T05:56:09,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-25T05:56:09,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742163_1339 (size=13119) 2024-11-25T05:56:09,260 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/e802cee469d44a22909ee199d82355ca as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/e802cee469d44a22909ee199d82355ca 2024-11-25T05:56:09,266 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f0a5da8d44418b4369e0cb022c813f46/C of f0a5da8d44418b4369e0cb022c813f46 into e802cee469d44a22909ee199d82355ca(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:56:09,266 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:56:09,266 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., storeName=f0a5da8d44418b4369e0cb022c813f46/C, priority=12, startTime=1732514169184; duration=0sec 2024-11-25T05:56:09,266 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:09,266 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0a5da8d44418b4369e0cb022c813f46:C 2024-11-25T05:56:09,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T05:56:09,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:09,288 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing f0a5da8d44418b4369e0cb022c813f46 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-25T05:56:09,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=A 2024-11-25T05:56:09,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:09,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=B 2024-11-25T05:56:09,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:09,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=C 2024-11-25T05:56:09,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:09,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/3f4c48398d284d348d35a6289babff11 is 50, key is test_row_0/A:col10/1732514167947/Put/seqid=0 2024-11-25T05:56:09,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742164_1340 (size=12301) 2024-11-25T05:56:09,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-25T05:56:09,707 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/3f4c48398d284d348d35a6289babff11 2024-11-25T05:56:09,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/a33bbb29b1a146b4b18a4add58baa6a1 is 50, key is test_row_0/B:col10/1732514167947/Put/seqid=0 2024-11-25T05:56:09,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742165_1341 (size=12301) 2024-11-25T05:56:09,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-25T05:56:10,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:56:10,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f0a5da8d44418b4369e0cb022c813f46 2024-11-25T05:56:10,118 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/a33bbb29b1a146b4b18a4add58baa6a1 2024-11-25T05:56:10,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/16d06fb0b9b1451daae5ad50880710ba is 50, key is test_row_0/C:col10/1732514167947/Put/seqid=0 2024-11-25T05:56:10,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742166_1342 (size=12301) 2024-11-25T05:56:10,129 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/16d06fb0b9b1451daae5ad50880710ba 2024-11-25T05:56:10,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/3f4c48398d284d348d35a6289babff11 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/3f4c48398d284d348d35a6289babff11 2024-11-25T05:56:10,135 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/3f4c48398d284d348d35a6289babff11, entries=150, sequenceid=518, filesize=12.0 K 2024-11-25T05:56:10,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/a33bbb29b1a146b4b18a4add58baa6a1 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/a33bbb29b1a146b4b18a4add58baa6a1 2024-11-25T05:56:10,140 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/a33bbb29b1a146b4b18a4add58baa6a1, entries=150, sequenceid=518, filesize=12.0 K 2024-11-25T05:56:10,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/16d06fb0b9b1451daae5ad50880710ba as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/16d06fb0b9b1451daae5ad50880710ba 2024-11-25T05:56:10,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732514170126, max timestamp=1732514170142Num uniques 859; 2024-11-25T05:56:10,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732514170126, max timestamp=1732514170142Num uniques 859; 2024-11-25T05:56:10,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-25T05:56:10,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-25T05:56:10,146 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732514170126, max timestamp=1732514170142Num uniques 859; 2024-11-25T05:56:10,146 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-25T05:56:10,150 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/16d06fb0b9b1451daae5ad50880710ba, entries=150, sequenceid=518, filesize=12.0 K 2024-11-25T05:56:10,151 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=60.38 KB/61830 for f0a5da8d44418b4369e0cb022c813f46 in 864ms, sequenceid=518, compaction requested=false 2024-11-25T05:56:10,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:56:10,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:10,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-25T05:56:10,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-25T05:56:10,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f0a5da8d44418b4369e0cb022c813f46 2024-11-25T05:56:10,154 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f0a5da8d44418b4369e0cb022c813f46 3/3 column families, dataSize=67.09 KB heapSize=176.39 KB 2024-11-25T05:56:10,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=A 2024-11-25T05:56:10,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:56:10,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=B 2024-11-25T05:56:10,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:56:10,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=C 2024-11-25T05:56:10,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:56:10,154 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-25T05:56:10,154 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0170 sec 2024-11-25T05:56:10,156 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.0220 sec 2024-11-25T05:56:10,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/93a88c6adb1d4cb6913fb644198a1d20 is 50, key is test_row_0/A:col10/1732514170146/Put/seqid=0 2024-11-25T05:56:10,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742167_1343 (size=14741) 2024-11-25T05:56:10,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:10,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514230186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:10,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:10,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:10,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:10,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:10,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514230191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:10,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:10,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:10,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:10,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:10,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514230192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:10,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:10,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:10,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:10,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-25T05:56:10,272 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:56:10,273 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:56:10,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-25T05:56:10,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-25T05:56:10,274 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:56:10,275 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:56:10,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:56:10,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:10,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514230302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:10,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:10,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:10,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:10,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514230302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:10,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:10,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:10,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:10,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:10,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:10,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514230303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:10,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:10,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:10,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:10,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-25T05:56:10,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-25T05:56:10,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:10,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:56:10,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:10,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:10,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:10,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:10,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:10,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514230512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:10,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:10,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:10,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:10,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:10,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514230512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:10,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:10,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:10,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514230513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:10,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:10,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:10,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:10,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:10,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:10,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=543 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/93a88c6adb1d4cb6913fb644198a1d20 2024-11-25T05:56:10,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/677740ddc98d4b21a79f21fcd99782cd is 50, key is test_row_0/B:col10/1732514170146/Put/seqid=0 2024-11-25T05:56:10,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742168_1344 (size=12301) 2024-11-25T05:56:10,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-25T05:56:10,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:10,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:56:10,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:10,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:10,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:10,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:10,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-25T05:56:10,708 INFO [Thread-1226 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1226. 2024-11-25T05:56:10,708 DEBUG [Thread-1226 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:56:10,708 DEBUG [Thread-1226 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:10,708 DEBUG [Thread-1226 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:10,709 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:56:10,709 INFO [Thread-1228 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1228. 2024-11-25T05:56:10,709 DEBUG [Thread-1228 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:56:10,709 DEBUG [Thread-1228 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:10,710 DEBUG [Thread-1228 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:10,710 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:56:10,711 INFO [Thread-1224 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1224. 2024-11-25T05:56:10,711 DEBUG [Thread-1224 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:56:10,711 DEBUG [Thread-1224 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:10,711 DEBUG [Thread-1224 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:10,711 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:56:10,713 INFO [Thread-1230 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1230. 2024-11-25T05:56:10,713 DEBUG [Thread-1230 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:56:10,713 DEBUG [Thread-1230 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:10,713 DEBUG [Thread-1230 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:10,713 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:56:10,714 INFO [Thread-1222 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1222. 2024-11-25T05:56:10,714 DEBUG [Thread-1222 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:56:10,714 DEBUG [Thread-1222 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:10,714 DEBUG [Thread-1222 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:10,715 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:56:10,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-25T05:56:10,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:10,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:56:10,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:10,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:10,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:10,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:10,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:10,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:10,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:10,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514230822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:10,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514230822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:10,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514230822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:10,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:10,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:10,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:10,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:10,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:10,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:10,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:10,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:10,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:10,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-25T05:56:10,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:10,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:56:10,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:10,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:10,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:10,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:10,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-25T05:56:10,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=543 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/677740ddc98d4b21a79f21fcd99782cd 2024-11-25T05:56:10,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/b634aa5e48c34699a70d91b25d2bf21e is 50, key is test_row_0/C:col10/1732514170146/Put/seqid=0 2024-11-25T05:56:10,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742169_1345 (size=12301) 2024-11-25T05:56:11,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-25T05:56:11,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:11,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:56:11,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:11,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:11,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:11,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-25T05:56:11,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:11,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:56:11,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:11,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:11,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:11,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:11,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:11,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:11,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:11,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53854 deadline: 1732514231332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:11,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53832 deadline: 1732514231332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:11,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53782 deadline: 1732514231332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:11,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:11,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:11,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:11,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:11,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:11,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:11,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:11,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:11,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:11,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-25T05:56:11,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:11,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:56:11,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:11,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:11,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:11,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:11,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=543 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/b634aa5e48c34699a70d91b25d2bf21e 2024-11-25T05:56:11,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/93a88c6adb1d4cb6913fb644198a1d20 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/93a88c6adb1d4cb6913fb644198a1d20 2024-11-25T05:56:11,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/93a88c6adb1d4cb6913fb644198a1d20, entries=200, sequenceid=543, filesize=14.4 K 2024-11-25T05:56:11,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/677740ddc98d4b21a79f21fcd99782cd as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/677740ddc98d4b21a79f21fcd99782cd 2024-11-25T05:56:11,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/677740ddc98d4b21a79f21fcd99782cd, entries=150, sequenceid=543, filesize=12.0 K 2024-11-25T05:56:11,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/b634aa5e48c34699a70d91b25d2bf21e as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/b634aa5e48c34699a70d91b25d2bf21e 2024-11-25T05:56:11,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/b634aa5e48c34699a70d91b25d2bf21e, entries=150, sequenceid=543, filesize=12.0 K 2024-11-25T05:56:11,411 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.34 KB/180576, currentSize=114.05 KB/116790 for f0a5da8d44418b4369e0cb022c813f46 in 1257ms, sequenceid=543, compaction requested=true 2024-11-25T05:56:11,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:56:11,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0a5da8d44418b4369e0cb022c813f46:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:56:11,411 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:56:11,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:11,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0a5da8d44418b4369e0cb022c813f46:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:56:11,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:11,411 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:56:11,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f0a5da8d44418b4369e0cb022c813f46:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:56:11,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:56:11,412 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:56:11,412 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:56:11,412 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): f0a5da8d44418b4369e0cb022c813f46/A is initiating minor compaction (all files) 2024-11-25T05:56:11,412 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): f0a5da8d44418b4369e0cb022c813f46/B is initiating minor compaction (all files) 2024-11-25T05:56:11,412 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0a5da8d44418b4369e0cb022c813f46/B in TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:11,412 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0a5da8d44418b4369e0cb022c813f46/A in TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:11,412 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/673395c53ff84f54a5fb3898e7572870, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/a33bbb29b1a146b4b18a4add58baa6a1, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/677740ddc98d4b21a79f21fcd99782cd] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp, totalSize=36.8 K 2024-11-25T05:56:11,412 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/988064dbdb2140d096bcfc20053fc029, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/3f4c48398d284d348d35a6289babff11, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/93a88c6adb1d4cb6913fb644198a1d20] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp, totalSize=39.2 K 2024-11-25T05:56:11,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-25T05:56:11,412 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 988064dbdb2140d096bcfc20053fc029, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1732514167613 2024-11-25T05:56:11,412 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 673395c53ff84f54a5fb3898e7572870, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1732514167613 2024-11-25T05:56:11,412 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f4c48398d284d348d35a6289babff11, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1732514167940 2024-11-25T05:56:11,413 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting a33bbb29b1a146b4b18a4add58baa6a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1732514167940 2024-11-25T05:56:11,413 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 93a88c6adb1d4cb6913fb644198a1d20, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=543, earliestPutTs=1732514170140 2024-11-25T05:56:11,413 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 677740ddc98d4b21a79f21fcd99782cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=543, earliestPutTs=1732514170146 2024-11-25T05:56:11,418 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0a5da8d44418b4369e0cb022c813f46#A#compaction#302 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:56:11,419 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/5ddb945d430641ae9d4d0de91bc6f001 is 50, key is test_row_0/A:col10/1732514170146/Put/seqid=0 2024-11-25T05:56:11,419 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0a5da8d44418b4369e0cb022c813f46#B#compaction#303 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:56:11,420 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/cc8785010fa74d6c9be6947ec52bcd1e is 50, key is test_row_0/B:col10/1732514170146/Put/seqid=0 2024-11-25T05:56:11,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742170_1346 (size=13221) 2024-11-25T05:56:11,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742171_1347 (size=13221) 2024-11-25T05:56:11,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-25T05:56:11,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:11,502 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing f0a5da8d44418b4369e0cb022c813f46 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-25T05:56:11,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=A 2024-11-25T05:56:11,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:11,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=B 2024-11-25T05:56:11,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:11,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=C 2024-11-25T05:56:11,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:11,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/236db22e65124e93adcbcbda65e29bf0 is 50, key is test_row_0/A:col10/1732514170183/Put/seqid=0 2024-11-25T05:56:11,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742172_1348 (size=12301) 2024-11-25T05:56:11,826 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/5ddb945d430641ae9d4d0de91bc6f001 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/5ddb945d430641ae9d4d0de91bc6f001 2024-11-25T05:56:11,826 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/cc8785010fa74d6c9be6947ec52bcd1e as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/cc8785010fa74d6c9be6947ec52bcd1e 2024-11-25T05:56:11,830 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0a5da8d44418b4369e0cb022c813f46/B of f0a5da8d44418b4369e0cb022c813f46 into cc8785010fa74d6c9be6947ec52bcd1e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:56:11,830 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0a5da8d44418b4369e0cb022c813f46/A of f0a5da8d44418b4369e0cb022c813f46 into 5ddb945d430641ae9d4d0de91bc6f001(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:56:11,831 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:56:11,831 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:56:11,831 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., storeName=f0a5da8d44418b4369e0cb022c813f46/B, priority=13, startTime=1732514171411; duration=0sec 2024-11-25T05:56:11,831 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., storeName=f0a5da8d44418b4369e0cb022c813f46/A, priority=13, startTime=1732514171411; duration=0sec 2024-11-25T05:56:11,831 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:56:11,831 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:56:11,831 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0a5da8d44418b4369e0cb022c813f46:B 2024-11-25T05:56:11,831 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0a5da8d44418b4369e0cb022c813f46:A 2024-11-25T05:56:11,831 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:56:11,832 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:56:11,832 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): f0a5da8d44418b4369e0cb022c813f46/C is initiating minor compaction (all files) 2024-11-25T05:56:11,832 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f0a5da8d44418b4369e0cb022c813f46/C in TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:11,832 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/e802cee469d44a22909ee199d82355ca, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/16d06fb0b9b1451daae5ad50880710ba, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/b634aa5e48c34699a70d91b25d2bf21e] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp, totalSize=36.8 K 2024-11-25T05:56:11,832 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting e802cee469d44a22909ee199d82355ca, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1732514167613 2024-11-25T05:56:11,833 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16d06fb0b9b1451daae5ad50880710ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1732514167940 2024-11-25T05:56:11,833 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting b634aa5e48c34699a70d91b25d2bf21e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=543, earliestPutTs=1732514170146 2024-11-25T05:56:11,839 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f0a5da8d44418b4369e0cb022c813f46#C#compaction#305 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:56:11,839 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/4975d1a422134626a2e84635b334548a is 50, key is test_row_0/C:col10/1732514170146/Put/seqid=0 2024-11-25T05:56:11,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742173_1349 (size=13221) 2024-11-25T05:56:11,911 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=563 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/236db22e65124e93adcbcbda65e29bf0 2024-11-25T05:56:11,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/741fe6294eef4b6fa91141d726059e81 is 50, key is test_row_0/B:col10/1732514170183/Put/seqid=0 2024-11-25T05:56:11,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742174_1350 (size=12301) 2024-11-25T05:56:12,248 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/4975d1a422134626a2e84635b334548a as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/4975d1a422134626a2e84635b334548a 2024-11-25T05:56:12,254 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f0a5da8d44418b4369e0cb022c813f46/C of f0a5da8d44418b4369e0cb022c813f46 into 4975d1a422134626a2e84635b334548a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:56:12,254 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:56:12,254 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., storeName=f0a5da8d44418b4369e0cb022c813f46/C, priority=13, startTime=1732514171411; duration=0sec 2024-11-25T05:56:12,254 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:12,254 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f0a5da8d44418b4369e0cb022c813f46:C 2024-11-25T05:56:12,322 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=563 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/741fe6294eef4b6fa91141d726059e81 2024-11-25T05:56:12,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/2ac57f449fef4905b69e7985f4fb6478 is 50, key is test_row_0/C:col10/1732514170183/Put/seqid=0 2024-11-25T05:56:12,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742175_1351 (size=12301) 2024-11-25T05:56:12,337 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=563 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/2ac57f449fef4905b69e7985f4fb6478 2024-11-25T05:56:12,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/236db22e65124e93adcbcbda65e29bf0 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/236db22e65124e93adcbcbda65e29bf0 2024-11-25T05:56:12,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on f0a5da8d44418b4369e0cb022c813f46 2024-11-25T05:56:12,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. as already flushing 2024-11-25T05:56:12,346 INFO [Thread-1219 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1219. 2024-11-25T05:56:12,346 INFO [Thread-1213 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1213. 2024-11-25T05:56:12,346 DEBUG [Thread-1219 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:56:12,346 DEBUG [Thread-1213 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:56:12,346 DEBUG [Thread-1219 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:12,346 DEBUG [Thread-1213 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:12,346 INFO [Thread-1211 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1211. 2024-11-25T05:56:12,346 DEBUG [Thread-1211 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:56:12,347 DEBUG [Thread-1211 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:12,347 DEBUG [Thread-1213 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:12,347 DEBUG [Thread-1219 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:12,347 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:56:12,347 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:56:12,347 DEBUG [Thread-1211 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:12,347 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:56:12,353 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/236db22e65124e93adcbcbda65e29bf0, entries=150, sequenceid=563, filesize=12.0 K 2024-11-25T05:56:12,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/741fe6294eef4b6fa91141d726059e81 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/741fe6294eef4b6fa91141d726059e81 2024-11-25T05:56:12,357 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/741fe6294eef4b6fa91141d726059e81, entries=150, sequenceid=563, filesize=12.0 K 2024-11-25T05:56:12,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/2ac57f449fef4905b69e7985f4fb6478 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/2ac57f449fef4905b69e7985f4fb6478 2024-11-25T05:56:12,368 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/2ac57f449fef4905b69e7985f4fb6478, entries=150, sequenceid=563, filesize=12.0 K 2024-11-25T05:56:12,369 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=20.13 KB/20610 for f0a5da8d44418b4369e0cb022c813f46 in 866ms, sequenceid=563, compaction requested=false 2024-11-25T05:56:12,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for f0a5da8d44418b4369e0cb022c813f46: 2024-11-25T05:56:12,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:12,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-25T05:56:12,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-25T05:56:12,372 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-25T05:56:12,372 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0950 sec 2024-11-25T05:56:12,374 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 2.1000 sec 2024-11-25T05:56:12,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-25T05:56:12,433 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:56:17,545 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T05:56:19,103 INFO [Thread-1215 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1215. 2024-11-25T05:56:19,104 DEBUG [Thread-1215 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:56:19,104 DEBUG [Thread-1215 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:19,104 DEBUG [Thread-1215 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:19,104 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:56:19,153 INFO [Thread-1217 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1217. 2024-11-25T05:56:19,153 DEBUG [Thread-1217 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:56:19,153 DEBUG [Thread-1217 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:19,153 DEBUG [Thread-1217 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:19,153 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:56:19,154 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-25T05:56:19,154 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-11-25T05:56:19,154 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 170 2024-11-25T05:56:19,154 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 7 2024-11-25T05:56:19,154 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 120 2024-11-25T05:56:19,154 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-11-25T05:56:19,154 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-25T05:56:19,154 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3154 2024-11-25T05:56:19,154 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3058 2024-11-25T05:56:19,154 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3162 2024-11-25T05:56:19,154 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3132 2024-11-25T05:56:19,154 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3024 2024-11-25T05:56:19,154 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-25T05:56:19,154 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T05:56:19,154 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T05:56:19,154 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:19,154 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:19,154 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:56:19,155 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-25T05:56:19,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-25T05:56:19,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-25T05:56:19,158 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732514179157"}]},"ts":"1732514179157"} 2024-11-25T05:56:19,159 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-25T05:56:19,159 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-25T05:56:19,160 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T05:56:19,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0a5da8d44418b4369e0cb022c813f46, UNASSIGN}] 2024-11-25T05:56:19,161 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0a5da8d44418b4369e0cb022c813f46, UNASSIGN 2024-11-25T05:56:19,162 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=f0a5da8d44418b4369e0cb022c813f46, regionState=CLOSING, regionLocation=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:19,163 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0a5da8d44418b4369e0cb022c813f46, UNASSIGN because future has completed 2024-11-25T05:56:19,163 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T05:56:19,163 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005}] 2024-11-25T05:56:19,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-25T05:56:19,315 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(122): Close f0a5da8d44418b4369e0cb022c813f46 2024-11-25T05:56:19,315 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T05:56:19,315 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1722): Closing f0a5da8d44418b4369e0cb022c813f46, disabling compactions & flushes 2024-11-25T05:56:19,315 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:19,315 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:19,315 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. after waiting 0 ms 2024-11-25T05:56:19,315 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:19,315 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(2902): Flushing f0a5da8d44418b4369e0cb022c813f46 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-25T05:56:19,315 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=A 2024-11-25T05:56:19,316 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:19,316 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=B 2024-11-25T05:56:19,316 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:19,316 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f0a5da8d44418b4369e0cb022c813f46, store=C 2024-11-25T05:56:19,316 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:19,319 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/3d081a6de411406eaedcc6a5e13be7e7 is 50, key is test_row_0/A:col10/1732514179152/Put/seqid=0 2024-11-25T05:56:19,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742176_1352 (size=9857) 2024-11-25T05:56:19,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-25T05:56:19,724 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/3d081a6de411406eaedcc6a5e13be7e7 2024-11-25T05:56:19,731 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/f0556e1d6d0a430fb90eb87a3488dfc6 is 50, key is test_row_0/B:col10/1732514179152/Put/seqid=0 2024-11-25T05:56:19,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742177_1353 (size=9857) 2024-11-25T05:56:19,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-25T05:56:20,139 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/f0556e1d6d0a430fb90eb87a3488dfc6 2024-11-25T05:56:20,145 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/7e50c459f760480aa4dcfa7b588a1275 is 50, key is test_row_0/C:col10/1732514179152/Put/seqid=0 2024-11-25T05:56:20,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742178_1354 (size=9857) 2024-11-25T05:56:20,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-25T05:56:20,549 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/7e50c459f760480aa4dcfa7b588a1275 2024-11-25T05:56:20,554 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/A/3d081a6de411406eaedcc6a5e13be7e7 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/3d081a6de411406eaedcc6a5e13be7e7 2024-11-25T05:56:20,557 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/3d081a6de411406eaedcc6a5e13be7e7, entries=100, sequenceid=574, filesize=9.6 K 2024-11-25T05:56:20,558 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/B/f0556e1d6d0a430fb90eb87a3488dfc6 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/f0556e1d6d0a430fb90eb87a3488dfc6 2024-11-25T05:56:20,562 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/f0556e1d6d0a430fb90eb87a3488dfc6, entries=100, sequenceid=574, filesize=9.6 K 2024-11-25T05:56:20,563 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/.tmp/C/7e50c459f760480aa4dcfa7b588a1275 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/7e50c459f760480aa4dcfa7b588a1275 2024-11-25T05:56:20,567 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/7e50c459f760480aa4dcfa7b588a1275, entries=100, sequenceid=574, filesize=9.6 K 2024-11-25T05:56:20,567 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f0a5da8d44418b4369e0cb022c813f46 in 1252ms, sequenceid=574, compaction requested=true 2024-11-25T05:56:20,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/725671e32bb24c1287ffd803aa6c2286, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/63a972f8d78a4687a43f3f465c649e55, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/4a26ccad7f38481fa949441676e4f3a5, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/3757c5f4997b41b2b85301ae5326a570, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/50da8401543e42689f507f272dd05ddf, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/aa008b3cbee34146ae4ced4193f41e4d, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/e83803859f0f4eec8a22e58eaf07e878, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/c912f17a31f24676854ad616d8a1f019, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/155a2edf9cae478280468220a0e3d9e6, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/6033f65298174e96befe685c1186a29f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/ec0062674b7241c792bc94530590ea05, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/27093543131b40e5b40b2f10f1034a62, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/c95b23e5fcff48adb3edbf68fb1fbd5e, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/51da812ae12a4385b17cd9d8a5f15acf, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/2f37435177d84f89ac9047b08c8c614a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/4647b2e34bc64cfb95bec2e76b6159b6, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/a03b67ba769248a69b2031e3c1abdc60, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/fa6c8dc1ae22495ca1ee2245bdc3961f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/910ea26dca8e480bbee9029d29ee6ef3, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/5d267b80fde3477799b0398c2cd1593f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/36292f53b08f4e96b288b546f6696ef8, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/9f4a262a0cb44a7ab2dc3fc6d0cd0984, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/135ccc15fd334f8193eb46fa7e9281ad, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/2b1a802aef56451ca2ae84e5e46fb34b, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/988064dbdb2140d096bcfc20053fc029, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/3f4c48398d284d348d35a6289babff11, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/93a88c6adb1d4cb6913fb644198a1d20] to archive 2024-11-25T05:56:20,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T05:56:20,570 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/725671e32bb24c1287ffd803aa6c2286 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/725671e32bb24c1287ffd803aa6c2286 2024-11-25T05:56:20,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/63a972f8d78a4687a43f3f465c649e55 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/63a972f8d78a4687a43f3f465c649e55 2024-11-25T05:56:20,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/4a26ccad7f38481fa949441676e4f3a5 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/4a26ccad7f38481fa949441676e4f3a5 2024-11-25T05:56:20,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/3757c5f4997b41b2b85301ae5326a570 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/3757c5f4997b41b2b85301ae5326a570 2024-11-25T05:56:20,575 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/50da8401543e42689f507f272dd05ddf to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/50da8401543e42689f507f272dd05ddf 2024-11-25T05:56:20,576 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/aa008b3cbee34146ae4ced4193f41e4d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/aa008b3cbee34146ae4ced4193f41e4d 2024-11-25T05:56:20,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/e83803859f0f4eec8a22e58eaf07e878 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/e83803859f0f4eec8a22e58eaf07e878 2024-11-25T05:56:20,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/c912f17a31f24676854ad616d8a1f019 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/c912f17a31f24676854ad616d8a1f019 2024-11-25T05:56:20,579 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/155a2edf9cae478280468220a0e3d9e6 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/155a2edf9cae478280468220a0e3d9e6 2024-11-25T05:56:20,581 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/6033f65298174e96befe685c1186a29f to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/6033f65298174e96befe685c1186a29f 2024-11-25T05:56:20,582 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/ec0062674b7241c792bc94530590ea05 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/ec0062674b7241c792bc94530590ea05 2024-11-25T05:56:20,583 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/27093543131b40e5b40b2f10f1034a62 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/27093543131b40e5b40b2f10f1034a62 2024-11-25T05:56:20,586 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/c95b23e5fcff48adb3edbf68fb1fbd5e to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/c95b23e5fcff48adb3edbf68fb1fbd5e 2024-11-25T05:56:20,588 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/51da812ae12a4385b17cd9d8a5f15acf to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/51da812ae12a4385b17cd9d8a5f15acf 2024-11-25T05:56:20,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/2f37435177d84f89ac9047b08c8c614a to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/2f37435177d84f89ac9047b08c8c614a 2024-11-25T05:56:20,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/4647b2e34bc64cfb95bec2e76b6159b6 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/4647b2e34bc64cfb95bec2e76b6159b6 2024-11-25T05:56:20,593 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/a03b67ba769248a69b2031e3c1abdc60 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/a03b67ba769248a69b2031e3c1abdc60 2024-11-25T05:56:20,594 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/fa6c8dc1ae22495ca1ee2245bdc3961f to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/fa6c8dc1ae22495ca1ee2245bdc3961f 2024-11-25T05:56:20,595 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/910ea26dca8e480bbee9029d29ee6ef3 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/910ea26dca8e480bbee9029d29ee6ef3 2024-11-25T05:56:20,596 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/5d267b80fde3477799b0398c2cd1593f to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/5d267b80fde3477799b0398c2cd1593f 2024-11-25T05:56:20,597 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/36292f53b08f4e96b288b546f6696ef8 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/36292f53b08f4e96b288b546f6696ef8 2024-11-25T05:56:20,598 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/9f4a262a0cb44a7ab2dc3fc6d0cd0984 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/9f4a262a0cb44a7ab2dc3fc6d0cd0984 2024-11-25T05:56:20,599 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/135ccc15fd334f8193eb46fa7e9281ad to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/135ccc15fd334f8193eb46fa7e9281ad 2024-11-25T05:56:20,600 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/2b1a802aef56451ca2ae84e5e46fb34b to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/2b1a802aef56451ca2ae84e5e46fb34b 2024-11-25T05:56:20,601 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/988064dbdb2140d096bcfc20053fc029 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/988064dbdb2140d096bcfc20053fc029 2024-11-25T05:56:20,602 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/3f4c48398d284d348d35a6289babff11 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/3f4c48398d284d348d35a6289babff11 2024-11-25T05:56:20,603 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/93a88c6adb1d4cb6913fb644198a1d20 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/93a88c6adb1d4cb6913fb644198a1d20 2024-11-25T05:56:20,605 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/ac5ea6159b33403fa06e01d7a221da7c, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/e71b59798f0444a8b757a454ca221f57, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/34de775130ae4a4780e6b93a0fe2ffbb, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/a5459840e88d4ace899d01152eaf187b, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/60961a8a5e0748a0bb66e22c5fd93810, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/5d8a4063a47b4a6b8ab8871a2f0767ca, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/f4f6fb10536642f89511ab9c44c0d113, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/bc9c85b072224d759c352f578ea393ae, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/a5eb4dd3d1074e5ca8fc0133afa1c603, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/d2629b7545764d46ac0311af0abacf5f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/5b2d1032593841e4a22569864862a69a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/b80da0051e8544efb4205de236a3c860, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/c12eb53113144272bc6373ac7d0680ed, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/ac86c0026b674301bba5cf4ac18e2a37, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/37629070169240e6ba441ac5c3369861, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/335a8b9df27f46f591b87f69e401c94b, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/3b2b015937cf484eb7986e8f4563acf7, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/183433453ef4468b966de4f98b678708, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/f75ccca4b45c41e3858ca3ea631b8b6f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/ccb4e5e4ab0840cf93a393f757a1f48f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/94d5a166f98a4f009afde0b3f0117c6f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/73d2d2a810ca4cdfb5a09a70e90ae1f0, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/d15d099b2b0d4ae29f791cc2dc8e8652, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/673395c53ff84f54a5fb3898e7572870, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/e195926933064cc28bdebfee6984cc11, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/a33bbb29b1a146b4b18a4add58baa6a1, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/677740ddc98d4b21a79f21fcd99782cd] to archive 2024-11-25T05:56:20,606 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T05:56:20,608 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/ac5ea6159b33403fa06e01d7a221da7c to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/ac5ea6159b33403fa06e01d7a221da7c 2024-11-25T05:56:20,609 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/e71b59798f0444a8b757a454ca221f57 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/e71b59798f0444a8b757a454ca221f57 2024-11-25T05:56:20,610 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/34de775130ae4a4780e6b93a0fe2ffbb to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/34de775130ae4a4780e6b93a0fe2ffbb 2024-11-25T05:56:20,611 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/a5459840e88d4ace899d01152eaf187b to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/a5459840e88d4ace899d01152eaf187b 2024-11-25T05:56:20,612 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/60961a8a5e0748a0bb66e22c5fd93810 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/60961a8a5e0748a0bb66e22c5fd93810 2024-11-25T05:56:20,614 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/5d8a4063a47b4a6b8ab8871a2f0767ca to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/5d8a4063a47b4a6b8ab8871a2f0767ca 2024-11-25T05:56:20,615 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/f4f6fb10536642f89511ab9c44c0d113 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/f4f6fb10536642f89511ab9c44c0d113 2024-11-25T05:56:20,617 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/bc9c85b072224d759c352f578ea393ae to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/bc9c85b072224d759c352f578ea393ae 2024-11-25T05:56:20,618 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/a5eb4dd3d1074e5ca8fc0133afa1c603 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/a5eb4dd3d1074e5ca8fc0133afa1c603 2024-11-25T05:56:20,619 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/d2629b7545764d46ac0311af0abacf5f to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/d2629b7545764d46ac0311af0abacf5f 2024-11-25T05:56:20,619 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/5b2d1032593841e4a22569864862a69a to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/5b2d1032593841e4a22569864862a69a 2024-11-25T05:56:20,620 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/b80da0051e8544efb4205de236a3c860 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/b80da0051e8544efb4205de236a3c860 2024-11-25T05:56:20,621 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/c12eb53113144272bc6373ac7d0680ed to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/c12eb53113144272bc6373ac7d0680ed 2024-11-25T05:56:20,622 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/ac86c0026b674301bba5cf4ac18e2a37 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/ac86c0026b674301bba5cf4ac18e2a37 2024-11-25T05:56:20,623 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/37629070169240e6ba441ac5c3369861 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/37629070169240e6ba441ac5c3369861 2024-11-25T05:56:20,624 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/335a8b9df27f46f591b87f69e401c94b to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/335a8b9df27f46f591b87f69e401c94b 2024-11-25T05:56:20,626 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/3b2b015937cf484eb7986e8f4563acf7 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/3b2b015937cf484eb7986e8f4563acf7 2024-11-25T05:56:20,627 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/183433453ef4468b966de4f98b678708 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/183433453ef4468b966de4f98b678708 2024-11-25T05:56:20,628 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/f75ccca4b45c41e3858ca3ea631b8b6f to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/f75ccca4b45c41e3858ca3ea631b8b6f 2024-11-25T05:56:20,629 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/ccb4e5e4ab0840cf93a393f757a1f48f to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/ccb4e5e4ab0840cf93a393f757a1f48f 2024-11-25T05:56:20,630 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/94d5a166f98a4f009afde0b3f0117c6f to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/94d5a166f98a4f009afde0b3f0117c6f 2024-11-25T05:56:20,631 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/73d2d2a810ca4cdfb5a09a70e90ae1f0 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/73d2d2a810ca4cdfb5a09a70e90ae1f0 2024-11-25T05:56:20,632 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/d15d099b2b0d4ae29f791cc2dc8e8652 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/d15d099b2b0d4ae29f791cc2dc8e8652 2024-11-25T05:56:20,633 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/673395c53ff84f54a5fb3898e7572870 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/673395c53ff84f54a5fb3898e7572870 2024-11-25T05:56:20,635 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/e195926933064cc28bdebfee6984cc11 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/e195926933064cc28bdebfee6984cc11 2024-11-25T05:56:20,637 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/a33bbb29b1a146b4b18a4add58baa6a1 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/a33bbb29b1a146b4b18a4add58baa6a1 2024-11-25T05:56:20,639 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/677740ddc98d4b21a79f21fcd99782cd to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/677740ddc98d4b21a79f21fcd99782cd 2024-11-25T05:56:20,641 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/3c95d4aade4e4bb1921e7e9f06cd0e9b, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/fdd15f89fa1946d5b306857901f01cd6, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/0adacd070f57473aa7a2b8d5426ce94d, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/32db3dbe001e4f0f98c7eeef0a3adb34, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/0a08a8d6189a451ca118a175e7094a06, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/daf5888e2bed4547a565bf6e88b0e677, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/5727d717d8a740e489459d4c78cae913, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/6f221ca9d5934dd8982be2a11b94d620, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/46baedff6db34a1eb5fecbd32dc24f60, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/081d283367d742c69408594f1dadb720, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/e6086f3d632c48e6b7a1d8d50d9385a4, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/69b1926e93e84f66b587b0d48d51f50b, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/f89472ffc181412c8729d3d2d29ace40, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/428e3cf0306447c5bfbe879d51cb2fe9, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/e06509eb75de4f30b3e640f0f7990f29, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/819d61e7efed40c2a4f1ef7f6683034a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/3ab1043331c2462b9d2af6b84eaebd6b, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/7d8077c019f7477fa7c2b74369109638, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/193596e43f4147e9b8f4dd6ce204e891, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/e01c5df790c0441098427d6cd15d4a81, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/2393570b5f764980b6e7353c40829802, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/dd3fcb5195db4a168a65460b4861b9cb, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/8bce6284ccd84207b662bf4208e53336, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/e802cee469d44a22909ee199d82355ca, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/550ab89aa5cd474485c8cef3401e2bf8, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/16d06fb0b9b1451daae5ad50880710ba, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/b634aa5e48c34699a70d91b25d2bf21e] to archive 2024-11-25T05:56:20,642 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T05:56:20,645 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/3c95d4aade4e4bb1921e7e9f06cd0e9b to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/3c95d4aade4e4bb1921e7e9f06cd0e9b 2024-11-25T05:56:20,649 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/fdd15f89fa1946d5b306857901f01cd6 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/fdd15f89fa1946d5b306857901f01cd6 2024-11-25T05:56:20,651 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/0adacd070f57473aa7a2b8d5426ce94d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/0adacd070f57473aa7a2b8d5426ce94d 2024-11-25T05:56:20,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/32db3dbe001e4f0f98c7eeef0a3adb34 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/32db3dbe001e4f0f98c7eeef0a3adb34 2024-11-25T05:56:20,658 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/0a08a8d6189a451ca118a175e7094a06 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/0a08a8d6189a451ca118a175e7094a06 2024-11-25T05:56:20,660 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/daf5888e2bed4547a565bf6e88b0e677 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/daf5888e2bed4547a565bf6e88b0e677 2024-11-25T05:56:20,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/5727d717d8a740e489459d4c78cae913 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/5727d717d8a740e489459d4c78cae913 2024-11-25T05:56:20,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/6f221ca9d5934dd8982be2a11b94d620 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/6f221ca9d5934dd8982be2a11b94d620 2024-11-25T05:56:20,667 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/46baedff6db34a1eb5fecbd32dc24f60 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/46baedff6db34a1eb5fecbd32dc24f60 2024-11-25T05:56:20,669 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/081d283367d742c69408594f1dadb720 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/081d283367d742c69408594f1dadb720 2024-11-25T05:56:20,671 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/e6086f3d632c48e6b7a1d8d50d9385a4 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/e6086f3d632c48e6b7a1d8d50d9385a4 2024-11-25T05:56:20,681 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/69b1926e93e84f66b587b0d48d51f50b to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/69b1926e93e84f66b587b0d48d51f50b 2024-11-25T05:56:20,682 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/f89472ffc181412c8729d3d2d29ace40 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/f89472ffc181412c8729d3d2d29ace40 2024-11-25T05:56:20,684 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/428e3cf0306447c5bfbe879d51cb2fe9 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/428e3cf0306447c5bfbe879d51cb2fe9 2024-11-25T05:56:20,685 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/e06509eb75de4f30b3e640f0f7990f29 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/e06509eb75de4f30b3e640f0f7990f29 2024-11-25T05:56:20,686 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/819d61e7efed40c2a4f1ef7f6683034a to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/819d61e7efed40c2a4f1ef7f6683034a 2024-11-25T05:56:20,687 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/3ab1043331c2462b9d2af6b84eaebd6b to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/3ab1043331c2462b9d2af6b84eaebd6b 2024-11-25T05:56:20,688 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/7d8077c019f7477fa7c2b74369109638 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/7d8077c019f7477fa7c2b74369109638 2024-11-25T05:56:20,689 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/193596e43f4147e9b8f4dd6ce204e891 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/193596e43f4147e9b8f4dd6ce204e891 2024-11-25T05:56:20,690 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/e01c5df790c0441098427d6cd15d4a81 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/e01c5df790c0441098427d6cd15d4a81 2024-11-25T05:56:20,691 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/2393570b5f764980b6e7353c40829802 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/2393570b5f764980b6e7353c40829802 2024-11-25T05:56:20,691 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/dd3fcb5195db4a168a65460b4861b9cb to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/dd3fcb5195db4a168a65460b4861b9cb 2024-11-25T05:56:20,692 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/8bce6284ccd84207b662bf4208e53336 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/8bce6284ccd84207b662bf4208e53336 2024-11-25T05:56:20,693 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/e802cee469d44a22909ee199d82355ca to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/e802cee469d44a22909ee199d82355ca 2024-11-25T05:56:20,694 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/550ab89aa5cd474485c8cef3401e2bf8 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/550ab89aa5cd474485c8cef3401e2bf8 2024-11-25T05:56:20,695 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/16d06fb0b9b1451daae5ad50880710ba to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/16d06fb0b9b1451daae5ad50880710ba 2024-11-25T05:56:20,696 DEBUG [StoreCloser-TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/b634aa5e48c34699a70d91b25d2bf21e to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/b634aa5e48c34699a70d91b25d2bf21e 2024-11-25T05:56:20,700 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/recovered.edits/577.seqid, newMaxSeqId=577, maxSeqId=1 2024-11-25T05:56:20,701 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. 2024-11-25T05:56:20,701 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1676): Region close journal for f0a5da8d44418b4369e0cb022c813f46: Waiting for close lock at 1732514179315Running coprocessor pre-close hooks at 1732514179315Disabling compacts and flushes for region at 1732514179315Disabling writes for close at 1732514179315Obtaining lock to block concurrent updates at 1732514179315Preparing flush snapshotting stores in f0a5da8d44418b4369e0cb022c813f46 at 1732514179315Finished memstore snapshotting TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732514179316 (+1 ms)Flushing stores of TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46. at 1732514179316Flushing f0a5da8d44418b4369e0cb022c813f46/A: creating writer at 1732514179316Flushing f0a5da8d44418b4369e0cb022c813f46/A: appending metadata at 1732514179319 (+3 ms)Flushing f0a5da8d44418b4369e0cb022c813f46/A: closing flushed file at 1732514179319Flushing f0a5da8d44418b4369e0cb022c813f46/B: creating writer at 1732514179728 (+409 ms)Flushing f0a5da8d44418b4369e0cb022c813f46/B: appending metadata at 1732514179730 (+2 ms)Flushing f0a5da8d44418b4369e0cb022c813f46/B: closing flushed file at 1732514179730Flushing f0a5da8d44418b4369e0cb022c813f46/C: creating writer at 1732514180143 (+413 ms)Flushing f0a5da8d44418b4369e0cb022c813f46/C: appending metadata at 1732514180145 (+2 ms)Flushing f0a5da8d44418b4369e0cb022c813f46/C: closing flushed file at 1732514180145Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b47d068: reopening flushed file at 1732514180553 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ecc9029: reopening flushed file at 1732514180557 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1692e99c: reopening flushed file at 1732514180562 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f0a5da8d44418b4369e0cb022c813f46 in 1252ms, sequenceid=574, compaction requested=true at 1732514180567 (+5 ms)Writing region close event to WAL at 1732514180697 (+130 ms)Running coprocessor post-close hooks at 1732514180700 (+3 ms)Closed at 1732514180701 (+1 ms) 2024-11-25T05:56:20,703 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(157): Closed f0a5da8d44418b4369e0cb022c813f46 2024-11-25T05:56:20,703 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=f0a5da8d44418b4369e0cb022c813f46, regionState=CLOSED 2024-11-25T05:56:20,705 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 because future has completed 2024-11-25T05:56:20,707 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-25T05:56:20,707 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure f0a5da8d44418b4369e0cb022c813f46, server=8ef925b832e3,35599,1732514089005 in 1.5430 sec 2024-11-25T05:56:20,708 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-25T05:56:20,708 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f0a5da8d44418b4369e0cb022c813f46, UNASSIGN in 1.5470 sec 2024-11-25T05:56:20,710 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-25T05:56:20,711 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5500 sec 2024-11-25T05:56:20,712 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732514180712"}]},"ts":"1732514180712"} 2024-11-25T05:56:20,713 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-25T05:56:20,713 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-25T05:56:20,715 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5590 sec 2024-11-25T05:56:21,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-25T05:56:21,303 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:56:21,304 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-25T05:56:21,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T05:56:21,309 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T05:56:21,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-25T05:56:21,310 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=84, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T05:56:21,314 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46 2024-11-25T05:56:21,317 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A, FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B, FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C, FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/recovered.edits] 2024-11-25T05:56:21,322 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/236db22e65124e93adcbcbda65e29bf0 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/236db22e65124e93adcbcbda65e29bf0 2024-11-25T05:56:21,324 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/3d081a6de411406eaedcc6a5e13be7e7 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/3d081a6de411406eaedcc6a5e13be7e7 2024-11-25T05:56:21,326 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/5ddb945d430641ae9d4d0de91bc6f001 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/A/5ddb945d430641ae9d4d0de91bc6f001 2024-11-25T05:56:21,329 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/741fe6294eef4b6fa91141d726059e81 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/741fe6294eef4b6fa91141d726059e81 2024-11-25T05:56:21,331 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/cc8785010fa74d6c9be6947ec52bcd1e to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/cc8785010fa74d6c9be6947ec52bcd1e 2024-11-25T05:56:21,333 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/f0556e1d6d0a430fb90eb87a3488dfc6 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/B/f0556e1d6d0a430fb90eb87a3488dfc6 2024-11-25T05:56:21,336 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/2ac57f449fef4905b69e7985f4fb6478 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/2ac57f449fef4905b69e7985f4fb6478 2024-11-25T05:56:21,337 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/4975d1a422134626a2e84635b334548a to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/4975d1a422134626a2e84635b334548a 2024-11-25T05:56:21,338 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/7e50c459f760480aa4dcfa7b588a1275 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/C/7e50c459f760480aa4dcfa7b588a1275 2024-11-25T05:56:21,341 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/recovered.edits/577.seqid to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46/recovered.edits/577.seqid 2024-11-25T05:56:21,342 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/f0a5da8d44418b4369e0cb022c813f46 2024-11-25T05:56:21,342 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-25T05:56:21,345 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=84, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T05:56:21,349 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-25T05:56:21,351 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-25T05:56:21,353 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=84, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T05:56:21,353 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-25T05:56:21,353 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732514181353"}]},"ts":"9223372036854775807"} 2024-11-25T05:56:21,355 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-25T05:56:21,355 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f0a5da8d44418b4369e0cb022c813f46, NAME => 'TestAcidGuarantees,,1732514148482.f0a5da8d44418b4369e0cb022c813f46.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T05:56:21,355 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-25T05:56:21,355 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732514181355"}]},"ts":"9223372036854775807"} 2024-11-25T05:56:21,358 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-25T05:56:21,358 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=84, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T05:56:21,360 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 53 msec 2024-11-25T05:56:21,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-25T05:56:21,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-25T05:56:21,423 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:56:21,430 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=247 (was 249), OpenFileDescriptor=451 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=411 (was 398) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7642 (was 8251) 2024-11-25T05:56:21,438 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=247, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=411, ProcessCount=11, AvailableMemoryMB=7642 2024-11-25T05:56:21,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T05:56:21,440 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T05:56:21,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-25T05:56:21,441 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T05:56:21,442 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:56:21,442 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 85 2024-11-25T05:56:21,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-25T05:56:21,442 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T05:56:21,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742179_1355 (size=1061) 2024-11-25T05:56:21,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-25T05:56:21,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-25T05:56:21,849 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a 2024-11-25T05:56:21,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742180_1356 (size=53) 2024-11-25T05:56:22,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-25T05:56:22,254 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T05:56:22,254 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ef2460dfb8241e76bc050e18474be77d, disabling compactions & flushes 2024-11-25T05:56:22,254 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:22,255 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:22,255 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. after waiting 0 ms 2024-11-25T05:56:22,255 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:22,255 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:22,255 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ef2460dfb8241e76bc050e18474be77d: Waiting for close lock at 1732514182254Disabling compacts and flushes for region at 1732514182254Disabling writes for close at 1732514182255 (+1 ms)Writing region close event to WAL at 1732514182255Closed at 1732514182255 2024-11-25T05:56:22,256 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T05:56:22,256 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732514182256"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732514182256"}]},"ts":"1732514182256"} 2024-11-25T05:56:22,257 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T05:56:22,258 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T05:56:22,258 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732514182258"}]},"ts":"1732514182258"} 2024-11-25T05:56:22,259 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-25T05:56:22,259 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef2460dfb8241e76bc050e18474be77d, ASSIGN}] 2024-11-25T05:56:22,260 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef2460dfb8241e76bc050e18474be77d, ASSIGN 2024-11-25T05:56:22,261 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef2460dfb8241e76bc050e18474be77d, ASSIGN; state=OFFLINE, location=8ef925b832e3,35599,1732514089005; forceNewPlan=false, retain=false 2024-11-25T05:56:22,412 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=ef2460dfb8241e76bc050e18474be77d, regionState=OPENING, regionLocation=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:22,414 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef2460dfb8241e76bc050e18474be77d, ASSIGN because future has completed 2024-11-25T05:56:22,414 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005}] 2024-11-25T05:56:22,569 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:22,569 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7752): Opening region: {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} 2024-11-25T05:56:22,569 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:22,569 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T05:56:22,569 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7794): checking encryption for ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:22,569 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7797): checking classloading for ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:22,570 INFO [StoreOpener-ef2460dfb8241e76bc050e18474be77d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:22,571 INFO [StoreOpener-ef2460dfb8241e76bc050e18474be77d-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T05:56:22,572 INFO [StoreOpener-ef2460dfb8241e76bc050e18474be77d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef2460dfb8241e76bc050e18474be77d columnFamilyName A 2024-11-25T05:56:22,572 DEBUG [StoreOpener-ef2460dfb8241e76bc050e18474be77d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:56:22,572 INFO [StoreOpener-ef2460dfb8241e76bc050e18474be77d-1 {}] regionserver.HStore(327): Store=ef2460dfb8241e76bc050e18474be77d/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T05:56:22,572 INFO [StoreOpener-ef2460dfb8241e76bc050e18474be77d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:22,573 INFO [StoreOpener-ef2460dfb8241e76bc050e18474be77d-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T05:56:22,573 INFO [StoreOpener-ef2460dfb8241e76bc050e18474be77d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef2460dfb8241e76bc050e18474be77d columnFamilyName B 2024-11-25T05:56:22,573 DEBUG [StoreOpener-ef2460dfb8241e76bc050e18474be77d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:56:22,574 INFO [StoreOpener-ef2460dfb8241e76bc050e18474be77d-1 {}] regionserver.HStore(327): Store=ef2460dfb8241e76bc050e18474be77d/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T05:56:22,574 INFO [StoreOpener-ef2460dfb8241e76bc050e18474be77d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:22,574 INFO [StoreOpener-ef2460dfb8241e76bc050e18474be77d-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T05:56:22,575 INFO [StoreOpener-ef2460dfb8241e76bc050e18474be77d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef2460dfb8241e76bc050e18474be77d columnFamilyName C 2024-11-25T05:56:22,575 DEBUG [StoreOpener-ef2460dfb8241e76bc050e18474be77d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:56:22,575 INFO [StoreOpener-ef2460dfb8241e76bc050e18474be77d-1 {}] regionserver.HStore(327): Store=ef2460dfb8241e76bc050e18474be77d/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T05:56:22,575 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:22,575 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1038): replaying wal for ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:22,576 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:22,576 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:22,577 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1048): stopping wal replay for ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:22,577 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1060): Cleaning up temporary data for ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:22,578 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T05:56:22,579 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1093): writing seq id for ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:22,581 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T05:56:22,581 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1114): Opened ef2460dfb8241e76bc050e18474be77d; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68850831, jitterRate=0.02595733106136322}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T05:56:22,581 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:22,582 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1006): Region open journal for ef2460dfb8241e76bc050e18474be77d: Running coprocessor pre-open hook at 1732514182570Writing region info on filesystem at 1732514182570Initializing all the Stores at 1732514182570Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732514182570Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732514182570Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732514182570Cleaning up temporary data from old regions at 1732514182577 (+7 ms)Running coprocessor post-open hooks at 1732514182581 (+4 ms)Region opened successfully at 1732514182582 (+1 ms) 2024-11-25T05:56:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-25T05:56:22,583 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., pid=87, masterSystemTime=1732514182566 2024-11-25T05:56:22,584 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:22,585 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:22,585 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=ef2460dfb8241e76bc050e18474be77d, regionState=OPEN, openSeqNum=2, regionLocation=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:22,586 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 because future has completed 2024-11-25T05:56:22,589 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-25T05:56:22,589 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 in 173 msec 2024-11-25T05:56:22,591 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-25T05:56:22,591 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef2460dfb8241e76bc050e18474be77d, ASSIGN in 330 msec 2024-11-25T05:56:22,592 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T05:56:22,592 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732514182592"}]},"ts":"1732514182592"} 2024-11-25T05:56:22,593 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-25T05:56:22,594 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T05:56:22,595 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1540 sec 2024-11-25T05:56:23,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-25T05:56:23,592 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:56:23,593 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ecc882, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:23,594 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:56:23,594 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:56:23,595 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:56:23,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:56:23,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:56:23,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@377afd8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:23,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:56:23,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:56:23,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:23,598 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33510, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:56:23,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66a1e7db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:23,599 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:56:23,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:56:23,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:56:23,602 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44896, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:56:23,602 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T05:56:23,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 8ef925b832e3,41285,1732514088402 2024-11-25T05:56:23,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@635d735e 2024-11-25T05:56:23,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T05:56:23,604 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33522, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T05:56:23,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T05:56:23,606 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T05:56:23,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-25T05:56:23,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742181_1357 (size=1097) 2024-11-25T05:56:24,021 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-25T05:56:24,022 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-25T05:56:24,027 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T05:56:24,031 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef2460dfb8241e76bc050e18474be77d, REOPEN/MOVE}] 2024-11-25T05:56:24,033 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef2460dfb8241e76bc050e18474be77d, REOPEN/MOVE 2024-11-25T05:56:24,034 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=ef2460dfb8241e76bc050e18474be77d, regionState=CLOSING, regionLocation=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:24,036 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef2460dfb8241e76bc050e18474be77d, REOPEN/MOVE because future has completed 2024-11-25T05:56:24,036 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T05:56:24,036 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005}] 2024-11-25T05:56:24,188 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:24,189 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T05:56:24,189 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing ef2460dfb8241e76bc050e18474be77d, disabling compactions & flushes 2024-11-25T05:56:24,189 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:24,189 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:24,189 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. after waiting 0 ms 2024-11-25T05:56:24,189 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:24,197 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T05:56:24,199 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:24,199 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for ef2460dfb8241e76bc050e18474be77d: Waiting for close lock at 1732514184189Running coprocessor pre-close hooks at 1732514184189Disabling compacts and flushes for region at 1732514184189Disabling writes for close at 1732514184189Writing region close event to WAL at 1732514184190 (+1 ms)Running coprocessor post-close hooks at 1732514184198 (+8 ms)Closed at 1732514184199 (+1 ms) 2024-11-25T05:56:24,199 WARN [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionServer(3299): Not adding moved region record: ef2460dfb8241e76bc050e18474be77d to self. 2024-11-25T05:56:24,202 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:24,203 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=ef2460dfb8241e76bc050e18474be77d, regionState=CLOSED 2024-11-25T05:56:24,205 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 because future has completed 2024-11-25T05:56:24,208 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-25T05:56:24,208 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 in 170 msec 2024-11-25T05:56:24,209 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef2460dfb8241e76bc050e18474be77d, REOPEN/MOVE; state=CLOSED, location=8ef925b832e3,35599,1732514089005; forceNewPlan=false, retain=true 2024-11-25T05:56:24,360 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=ef2460dfb8241e76bc050e18474be77d, regionState=OPENING, regionLocation=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:24,362 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef2460dfb8241e76bc050e18474be77d, REOPEN/MOVE because future has completed 2024-11-25T05:56:24,362 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005}] 2024-11-25T05:56:24,518 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:24,518 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7752): Opening region: {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} 2024-11-25T05:56:24,518 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:24,518 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T05:56:24,519 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7794): checking encryption for ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:24,519 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7797): checking classloading for ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:24,521 INFO [StoreOpener-ef2460dfb8241e76bc050e18474be77d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:24,522 INFO [StoreOpener-ef2460dfb8241e76bc050e18474be77d-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T05:56:24,522 INFO [StoreOpener-ef2460dfb8241e76bc050e18474be77d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef2460dfb8241e76bc050e18474be77d columnFamilyName A 2024-11-25T05:56:24,524 DEBUG [StoreOpener-ef2460dfb8241e76bc050e18474be77d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:56:24,525 INFO [StoreOpener-ef2460dfb8241e76bc050e18474be77d-1 {}] regionserver.HStore(327): Store=ef2460dfb8241e76bc050e18474be77d/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T05:56:24,526 INFO [StoreOpener-ef2460dfb8241e76bc050e18474be77d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:24,527 INFO [StoreOpener-ef2460dfb8241e76bc050e18474be77d-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T05:56:24,527 INFO [StoreOpener-ef2460dfb8241e76bc050e18474be77d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef2460dfb8241e76bc050e18474be77d columnFamilyName B 2024-11-25T05:56:24,527 DEBUG [StoreOpener-ef2460dfb8241e76bc050e18474be77d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:56:24,528 INFO [StoreOpener-ef2460dfb8241e76bc050e18474be77d-1 {}] regionserver.HStore(327): Store=ef2460dfb8241e76bc050e18474be77d/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T05:56:24,528 INFO [StoreOpener-ef2460dfb8241e76bc050e18474be77d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:24,529 INFO [StoreOpener-ef2460dfb8241e76bc050e18474be77d-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T05:56:24,529 INFO [StoreOpener-ef2460dfb8241e76bc050e18474be77d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef2460dfb8241e76bc050e18474be77d columnFamilyName C 2024-11-25T05:56:24,529 DEBUG [StoreOpener-ef2460dfb8241e76bc050e18474be77d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:56:24,529 INFO [StoreOpener-ef2460dfb8241e76bc050e18474be77d-1 {}] regionserver.HStore(327): Store=ef2460dfb8241e76bc050e18474be77d/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T05:56:24,530 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:24,530 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1038): replaying wal for ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:24,531 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:24,532 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:24,534 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1048): stopping wal replay for ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:24,534 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1060): Cleaning up temporary data for ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:24,535 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T05:56:24,537 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1093): writing seq id for ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:24,538 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1114): Opened ef2460dfb8241e76bc050e18474be77d; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74283647, jitterRate=0.10691259801387787}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T05:56:24,538 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:24,539 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1006): Region open journal for ef2460dfb8241e76bc050e18474be77d: Running coprocessor pre-open hook at 1732514184519Writing region info on filesystem at 1732514184519Initializing all the Stores at 1732514184520 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732514184520Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732514184520Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732514184520Cleaning up temporary data from old regions at 1732514184534 (+14 ms)Running coprocessor post-open hooks at 1732514184538 (+4 ms)Region opened successfully at 1732514184539 (+1 ms) 2024-11-25T05:56:24,540 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., pid=92, masterSystemTime=1732514184513 2024-11-25T05:56:24,543 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:24,543 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:24,544 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=ef2460dfb8241e76bc050e18474be77d, regionState=OPEN, openSeqNum=5, regionLocation=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:24,545 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 because future has completed 2024-11-25T05:56:24,549 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=90 2024-11-25T05:56:24,549 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 in 184 msec 2024-11-25T05:56:24,550 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-25T05:56:24,550 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef2460dfb8241e76bc050e18474be77d, REOPEN/MOVE in 518 msec 2024-11-25T05:56:24,552 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-25T05:56:24,552 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 524 msec 2024-11-25T05:56:24,554 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 947 msec 2024-11-25T05:56:24,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-25T05:56:24,555 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:56:24,556 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15f86bdc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:24,556 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:56:24,556 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:56:24,557 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:56:24,557 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:56:24,557 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:56:24,557 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a92efe2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:24,557 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:56:24,557 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:56:24,557 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:24,558 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33544, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:56:24,558 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b6011f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:24,559 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b10b9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:24,559 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:56:24,559 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:56:24,559 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:56:24,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:56:24,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:56:24,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76c378aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:24,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:56:24,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:56:24,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:24,560 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33558, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:56:24,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15d0bb31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:24,562 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c265f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:24,562 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:56:24,562 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:56:24,562 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:56:24,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:56:24,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:56:24,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23b79956, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:24,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:56:24,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:56:24,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:24,564 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33578, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:56:24,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bf7c978, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:24,565 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c21b87a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:24,565 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:56:24,565 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:56:24,565 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:56:24,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:56:24,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:56:24,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fa46de4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:24,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:56:24,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:56:24,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:24,566 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:56:24,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dd69848, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:24,567 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35e136ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:24,567 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:56:24,567 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:56:24,568 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:56:24,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:56:24,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:56:24,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6abbbb41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:24,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:56:24,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:56:24,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:24,569 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33632, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:56:24,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c947352, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:24,570 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72c86b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:24,570 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:56:24,570 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:56:24,571 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:56:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:56:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:56:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7779e4c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:56:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:56:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:24,572 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33638, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:56:24,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32701511, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:24,573 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35aa2c8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:24,573 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:56:24,573 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:56:24,574 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:56:24,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:56:24,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:56:24,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9e9d2a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:24,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:56:24,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:56:24,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:24,575 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33660, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:56:24,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7369e6ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:24,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f9edfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:24,576 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:56:24,576 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:56:24,577 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:56:24,577 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:56:24,577 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:56:24,577 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45c577b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:24,578 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:56:24,578 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:56:24,578 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:24,579 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33670, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:56:24,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@295a60a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:24,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42125f21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:24,580 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:56:24,580 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:56:24,581 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:56:24,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:56:24,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:56:24,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a829be1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:24,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:56:24,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:56:24,582 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:24,582 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33678, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:56:24,583 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@258245c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:24,583 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b126a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:24,584 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:56:24,584 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:56:24,584 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:56:24,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:56:24,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:56:24,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2336b97d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:24,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:56:24,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:56:24,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:24,586 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33706, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:56:24,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33abd8f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:24,587 DEBUG [Thread-1618 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:56:24,587 DEBUG [Thread-1623 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:56:24,587 DEBUG [Thread-1627 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:56:24,587 DEBUG [Thread-1631 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:56:24,588 DEBUG [Thread-1625 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:56:24,588 DEBUG [Thread-1616 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:56:24,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:56:24,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:56:24,588 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:56:24,589 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:56:24,589 DEBUG [Thread-1614 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:56:24,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:56:24,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:56:24,589 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:56:24,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:56:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:56:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:56:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:56:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:56:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:56:24,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-25T05:56:24,590 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44910, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:56:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-25T05:56:24,591 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44916, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:56:24,591 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44934, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:56:24,591 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44924, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:56:24,592 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:56:24,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:56:24,593 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:56:24,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5] 2024-11-25T05:56:24,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:56:24,593 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:56:24,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5] 2024-11-25T05:56:24,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5] 2024-11-25T05:56:24,594 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44954, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:56:24,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5] 2024-11-25T05:56:24,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5] 2024-11-25T05:56:24,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5] 2024-11-25T05:56:24,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5] 2024-11-25T05:56:24,597 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:56:24,598 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:56:24,606 DEBUG [Thread-1629 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:56:24,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:56:24,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:56:24,608 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:56:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:24,609 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ef2460dfb8241e76bc050e18474be77d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-25T05:56:24,609 DEBUG [Thread-1612 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:56:24,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5] 2024-11-25T05:56:24,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=A 2024-11-25T05:56:24,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:56:24,611 DEBUG [Thread-1620 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:56:24,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:24,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=B 2024-11-25T05:56:24,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:24,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=C 2024-11-25T05:56:24,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:24,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:56:24,612 DEBUG [Thread-1620 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:56:24,612 DEBUG [Thread-1620 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:56:24,613 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44984, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:56:24,614 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44986, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:56:24,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5] 2024-11-25T05:56:24,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5] 2024-11-25T05:56:24,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125a280c2e976724bb9ae2824ddeb0378f0_ef2460dfb8241e76bc050e18474be77d is 50, key is test_row_0/A:col10/1732514184608/Put/seqid=0 2024-11-25T05:56:24,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742182_1358 (size=14594) 2024-11-25T05:56:24,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:24,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44934 deadline: 1732514244664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:24,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:24,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:24,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:24,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:24,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44954 deadline: 1732514244669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:24,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732514184658, max timestamp=1732514184671Num uniques 859; 2024-11-25T05:56:24,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732514184647, max timestamp=1732514184671Num uniques 859; 2024-11-25T05:56:24,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-25T05:56:24,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-25T05:56:24,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732514184647, max timestamp=1732514184671Num uniques 859; 2024-11-25T05:56:24,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:24,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-25T05:56:24,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:24,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:24,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-25T05:56:24,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:24,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514244698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:24,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:24,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:24,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:24,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514244699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:24,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44986 deadline: 1732514244703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:24,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:24,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:24,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:24,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:24,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:24,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:24,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-25T05:56:24,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:24,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:24,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:24,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:24,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:24,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:24,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44934 deadline: 1732514244782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:24,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:24,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:24,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:24,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:24,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44954 deadline: 1732514244782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:24,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:24,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:24,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:24,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514244811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:24,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514244822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:24,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:24,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:24,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44986 deadline: 1732514244822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:24,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:24,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:24,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:24,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-25T05:56:24,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-25T05:56:24,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:24,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:24,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:24,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:24,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:25,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:25,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44934 deadline: 1732514244992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:25,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:25,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:25,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:25,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:25,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44954 deadline: 1732514244992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:25,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:25,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:25,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:25,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:25,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514245022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:25,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:25,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:25,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:25,042 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:56:25,047 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125a280c2e976724bb9ae2824ddeb0378f0_ef2460dfb8241e76bc050e18474be77d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125a280c2e976724bb9ae2824ddeb0378f0_ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:25,048 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/04f0663834584962b1b5c03efefb68a6, store: [table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:25,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/04f0663834584962b1b5c03efefb68a6 is 175, key is test_row_0/A:col10/1732514184608/Put/seqid=0 2024-11-25T05:56:25,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:25,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514245042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:25,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:25,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:25,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44986 deadline: 1732514245043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:25,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:25,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:25,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:25,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:25,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:25,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742183_1359 (size=39549) 2024-11-25T05:56:25,054 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/04f0663834584962b1b5c03efefb68a6 2024-11-25T05:56:25,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-25T05:56:25,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:25,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:25,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:25,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:25,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:25,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:25,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/1d7af0ce5cab4b7e9fc2120035701417 is 50, key is test_row_0/B:col10/1732514184608/Put/seqid=0 2024-11-25T05:56:25,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742184_1360 (size=12001) 2024-11-25T05:56:25,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-25T05:56:25,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:25,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:25,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:25,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:25,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:25,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:25,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-25T05:56:25,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:25,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44934 deadline: 1732514245312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:25,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:25,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:25,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:25,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:25,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44954 deadline: 1732514245312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:25,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:25,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:25,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:25,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:25,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514245332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:25,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:25,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:25,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:25,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-25T05:56:25,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:25,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:25,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:25,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:25,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:25,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:25,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:25,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514245361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:25,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:25,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:25,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44986 deadline: 1732514245362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:25,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:25,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:25,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:25,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:25,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:25,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/1d7af0ce5cab4b7e9fc2120035701417 2024-11-25T05:56:25,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/5b87bc3d74ec409e9fe5d4f2a4bc195b is 50, key is test_row_0/C:col10/1732514184608/Put/seqid=0 2024-11-25T05:56:25,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742185_1361 (size=12001) 2024-11-25T05:56:25,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/5b87bc3d74ec409e9fe5d4f2a4bc195b 2024-11-25T05:56:25,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/04f0663834584962b1b5c03efefb68a6 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/04f0663834584962b1b5c03efefb68a6 2024-11-25T05:56:25,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-25T05:56:25,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:25,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:25,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:25,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:25,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:25,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:25,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/04f0663834584962b1b5c03efefb68a6, entries=200, sequenceid=15, filesize=38.6 K 2024-11-25T05:56:25,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/1d7af0ce5cab4b7e9fc2120035701417 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/1d7af0ce5cab4b7e9fc2120035701417 2024-11-25T05:56:25,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/1d7af0ce5cab4b7e9fc2120035701417, entries=150, sequenceid=15, filesize=11.7 K 2024-11-25T05:56:25,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/5b87bc3d74ec409e9fe5d4f2a4bc195b as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/5b87bc3d74ec409e9fe5d4f2a4bc195b 2024-11-25T05:56:25,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/5b87bc3d74ec409e9fe5d4f2a4bc195b, entries=150, sequenceid=15, filesize=11.7 K 2024-11-25T05:56:25,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=131.94 KB/135110 for ef2460dfb8241e76bc050e18474be77d in 921ms, sequenceid=15, compaction requested=false 2024-11-25T05:56:25,530 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-25T05:56:25,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ef2460dfb8241e76bc050e18474be77d: 2024-11-25T05:56:25,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-25T05:56:25,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:25,668 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing ef2460dfb8241e76bc050e18474be77d 3/3 column families, dataSize=131.94 KB heapSize=361.16 KB 2024-11-25T05:56:25,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=A 2024-11-25T05:56:25,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:56:25,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=B 2024-11-25T05:56:25,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:56:25,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=C 2024-11-25T05:56:25,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:56:25,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112562780183490b416e8afee077f0a599ea_ef2460dfb8241e76bc050e18474be77d is 50, key is test_row_0/A:col10/1732514184701/Put/seqid=0 2024-11-25T05:56:25,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742186_1362 (size=12154) 2024-11-25T05:56:25,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:56:25,686 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112562780183490b416e8afee077f0a599ea_ef2460dfb8241e76bc050e18474be77d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112562780183490b416e8afee077f0a599ea_ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:25,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/c629c432b45245b1a8b6b4bdbdea793f, store: [table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:25,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/c629c432b45245b1a8b6b4bdbdea793f is 175, key is test_row_0/A:col10/1732514184701/Put/seqid=0 2024-11-25T05:56:25,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742187_1363 (size=30955) 2024-11-25T05:56:25,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-25T05:56:25,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:25,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:25,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:25,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44954 deadline: 1732514245851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:25,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:25,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:25,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:25,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514245852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:25,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:25,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:25,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:25,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:25,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:25,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44934 deadline: 1732514245854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:25,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:25,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:25,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:25,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:25,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514245872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:25,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:25,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:25,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:25,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44986 deadline: 1732514245872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:25,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:25,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:25,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:25,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:25,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:25,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44954 deadline: 1732514245962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:25,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:25,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:25,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:25,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:25,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44934 deadline: 1732514245972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:25,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:25,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:25,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:26,092 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=48, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/c629c432b45245b1a8b6b4bdbdea793f 2024-11-25T05:56:26,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/f7bde89c7cd641fb81ce79f7813b59a2 is 50, key is test_row_0/B:col10/1732514184701/Put/seqid=0 2024-11-25T05:56:26,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742188_1364 (size=12001) 2024-11-25T05:56:26,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:26,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44954 deadline: 1732514246172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:26,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:26,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:26,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:26,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:26,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44934 deadline: 1732514246182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:26,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:26,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:26,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:26,247 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T05:56:26,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:26,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44954 deadline: 1732514246492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:26,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:26,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:26,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:26,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44934 deadline: 1732514246492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:26,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:26,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:26,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:26,503 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/f7bde89c7cd641fb81ce79f7813b59a2 2024-11-25T05:56:26,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/add9b508983046dbace60a552116a048 is 50, key is test_row_0/C:col10/1732514184701/Put/seqid=0 2024-11-25T05:56:26,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742189_1365 (size=12001) 2024-11-25T05:56:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-25T05:56:26,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:26,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514246872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:26,878 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_2' in ef2460dfb8241e76bc050e18474be77d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:56:26,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:26,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:26,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:26,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514246882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:26,889 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_0' in ef2460dfb8241e76bc050e18474be77d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:56:26,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:26,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:26,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:26,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:26,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44986 deadline: 1732514246891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:26,902 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_1' in ef2460dfb8241e76bc050e18474be77d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:56:26,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:26,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:26,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:26,914 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/add9b508983046dbace60a552116a048 2024-11-25T05:56:26,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/c629c432b45245b1a8b6b4bdbdea793f as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/c629c432b45245b1a8b6b4bdbdea793f 2024-11-25T05:56:26,921 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/c629c432b45245b1a8b6b4bdbdea793f, entries=150, sequenceid=48, filesize=30.2 K 2024-11-25T05:56:26,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/f7bde89c7cd641fb81ce79f7813b59a2 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/f7bde89c7cd641fb81ce79f7813b59a2 2024-11-25T05:56:26,925 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/f7bde89c7cd641fb81ce79f7813b59a2, entries=150, sequenceid=48, filesize=11.7 K 2024-11-25T05:56:26,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/add9b508983046dbace60a552116a048 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/add9b508983046dbace60a552116a048 2024-11-25T05:56:26,930 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/add9b508983046dbace60a552116a048, entries=150, sequenceid=48, filesize=11.7 K 2024-11-25T05:56:26,931 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~361.11 KB/369776, currentSize=53.67 KB/54960 for ef2460dfb8241e76bc050e18474be77d in 1263ms, sequenceid=48, compaction requested=false 2024-11-25T05:56:26,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for ef2460dfb8241e76bc050e18474be77d: 2024-11-25T05:56:26,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:26,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-25T05:56:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-25T05:56:26,934 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-25T05:56:26,934 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3340 sec 2024-11-25T05:56:26,936 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 2.3460 sec 2024-11-25T05:56:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:27,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ef2460dfb8241e76bc050e18474be77d 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-25T05:56:27,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=A 2024-11-25T05:56:27,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:27,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=B 2024-11-25T05:56:27,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:27,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=C 2024-11-25T05:56:27,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:27,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c14d19322bf74ad3a5d7e39ff6e993cf_ef2460dfb8241e76bc050e18474be77d is 50, key is test_row_0/A:col10/1732514187004/Put/seqid=0 2024-11-25T05:56:27,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742190_1366 (size=14594) 2024-11-25T05:56:27,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44954 deadline: 1732514247061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:27,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:27,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:27,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:27,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44934 deadline: 1732514247062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:27,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:27,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44954 deadline: 1732514247172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:27,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:27,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:27,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:27,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44934 deadline: 1732514247172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:27,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:27,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:27,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:27,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:27,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44954 deadline: 1732514247382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:27,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:27,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:27,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:27,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:27,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44934 deadline: 1732514247382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:27,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:27,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:27,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:27,416 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:56:27,419 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c14d19322bf74ad3a5d7e39ff6e993cf_ef2460dfb8241e76bc050e18474be77d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c14d19322bf74ad3a5d7e39ff6e993cf_ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:27,420 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/fcaaad4739ca4a0094a63b41e0c44b7e, store: [table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:27,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/fcaaad4739ca4a0094a63b41e0c44b7e is 175, key is test_row_0/A:col10/1732514187004/Put/seqid=0 2024-11-25T05:56:27,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742191_1367 (size=39549) 2024-11-25T05:56:27,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:27,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44954 deadline: 1732514247702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:27,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:27,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:27,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:27,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:27,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44934 deadline: 1732514247702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:27,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:27,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:27,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:27,824 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/fcaaad4739ca4a0094a63b41e0c44b7e 2024-11-25T05:56:27,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/6e8a2ee8ddde44798964b44d420d324d is 50, key is test_row_0/B:col10/1732514187004/Put/seqid=0 2024-11-25T05:56:27,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742192_1368 (size=12001) 2024-11-25T05:56:28,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:28,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44954 deadline: 1732514248212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:28,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:28,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:28,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:28,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:28,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44934 deadline: 1732514248222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:28,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:28,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:28,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:28,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/6e8a2ee8ddde44798964b44d420d324d 2024-11-25T05:56:28,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/698c56d0f6594c7c8eb3cde6a547678f is 50, key is test_row_0/C:col10/1732514187004/Put/seqid=0 2024-11-25T05:56:28,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742193_1369 (size=12001) 2024-11-25T05:56:28,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/698c56d0f6594c7c8eb3cde6a547678f 2024-11-25T05:56:28,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/fcaaad4739ca4a0094a63b41e0c44b7e as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/fcaaad4739ca4a0094a63b41e0c44b7e 2024-11-25T05:56:28,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/fcaaad4739ca4a0094a63b41e0c44b7e, entries=200, sequenceid=60, filesize=38.6 K 2024-11-25T05:56:28,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/6e8a2ee8ddde44798964b44d420d324d as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/6e8a2ee8ddde44798964b44d420d324d 2024-11-25T05:56:28,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/6e8a2ee8ddde44798964b44d420d324d, entries=150, sequenceid=60, filesize=11.7 K 2024-11-25T05:56:28,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/698c56d0f6594c7c8eb3cde6a547678f as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/698c56d0f6594c7c8eb3cde6a547678f 2024-11-25T05:56:28,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/698c56d0f6594c7c8eb3cde6a547678f, entries=150, sequenceid=60, filesize=11.7 K 2024-11-25T05:56:28,668 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ef2460dfb8241e76bc050e18474be77d in 1662ms, sequenceid=60, compaction requested=true 2024-11-25T05:56:28,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ef2460dfb8241e76bc050e18474be77d: 2024-11-25T05:56:28,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef2460dfb8241e76bc050e18474be77d:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:56:28,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:28,668 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:56:28,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef2460dfb8241e76bc050e18474be77d:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:56:28,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:28,668 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:56:28,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef2460dfb8241e76bc050e18474be77d:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:56:28,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:56:28,669 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:56:28,669 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:56:28,669 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): ef2460dfb8241e76bc050e18474be77d/B is initiating minor compaction (all files) 2024-11-25T05:56:28,669 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): ef2460dfb8241e76bc050e18474be77d/A is initiating minor compaction (all files) 2024-11-25T05:56:28,669 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef2460dfb8241e76bc050e18474be77d/A in TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:28,669 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef2460dfb8241e76bc050e18474be77d/B in TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:28,669 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/04f0663834584962b1b5c03efefb68a6, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/c629c432b45245b1a8b6b4bdbdea793f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/fcaaad4739ca4a0094a63b41e0c44b7e] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp, totalSize=107.5 K 2024-11-25T05:56:28,669 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/1d7af0ce5cab4b7e9fc2120035701417, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/f7bde89c7cd641fb81ce79f7813b59a2, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/6e8a2ee8ddde44798964b44d420d324d] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp, totalSize=35.2 K 2024-11-25T05:56:28,669 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:28,669 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. files: [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/04f0663834584962b1b5c03efefb68a6, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/c629c432b45245b1a8b6b4bdbdea793f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/fcaaad4739ca4a0094a63b41e0c44b7e] 2024-11-25T05:56:28,670 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d7af0ce5cab4b7e9fc2120035701417, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732514184599 2024-11-25T05:56:28,670 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04f0663834584962b1b5c03efefb68a6, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732514184599 2024-11-25T05:56:28,670 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting f7bde89c7cd641fb81ce79f7813b59a2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732514184691 2024-11-25T05:56:28,670 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting c629c432b45245b1a8b6b4bdbdea793f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732514184691 2024-11-25T05:56:28,670 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e8a2ee8ddde44798964b44d420d324d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732514185850 2024-11-25T05:56:28,670 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcaaad4739ca4a0094a63b41e0c44b7e, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732514185840 2024-11-25T05:56:28,675 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:28,676 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125f22355c326fc41149bb9c9ec22017d32_ef2460dfb8241e76bc050e18474be77d store=[table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:28,677 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef2460dfb8241e76bc050e18474be77d#B#compaction#321 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:56:28,677 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/8b74f235df864c4daa15470f9cb725ab is 50, key is test_row_0/B:col10/1732514187004/Put/seqid=0 2024-11-25T05:56:28,679 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125f22355c326fc41149bb9c9ec22017d32_ef2460dfb8241e76bc050e18474be77d, store=[table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:28,679 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125f22355c326fc41149bb9c9ec22017d32_ef2460dfb8241e76bc050e18474be77d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:28,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742194_1370 (size=12104) 2024-11-25T05:56:28,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742195_1371 (size=4469) 2024-11-25T05:56:28,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-25T05:56:28,752 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:56:28,753 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:56:28,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-25T05:56:28,755 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:56:28,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-25T05:56:28,756 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:56:28,756 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:56:28,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-25T05:56:28,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:28,897 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ef2460dfb8241e76bc050e18474be77d 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-25T05:56:28,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=A 2024-11-25T05:56:28,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:56:28,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=B 2024-11-25T05:56:28,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:56:28,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=C 2024-11-25T05:56:28,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:56:28,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112556b41fdb7335446e8f5c5c3bf51da7f0_ef2460dfb8241e76bc050e18474be77d is 50, key is test_row_0/A:col10/1732514187053/Put/seqid=0 2024-11-25T05:56:28,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-25T05:56:28,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:28,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:28,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:28,910 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:28,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:28,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:28,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742196_1372 (size=14594) 2024-11-25T05:56:28,912 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:56:28,915 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112556b41fdb7335446e8f5c5c3bf51da7f0_ef2460dfb8241e76bc050e18474be77d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112556b41fdb7335446e8f5c5c3bf51da7f0_ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:28,916 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/979e8d987041446c96c1222d08e759ae, store: [table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:28,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/979e8d987041446c96c1222d08e759ae is 175, key is test_row_0/A:col10/1732514187053/Put/seqid=0 2024-11-25T05:56:28,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742197_1373 (size=39549) 2024-11-25T05:56:28,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:28,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514248932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:28,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:28,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:28,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:28,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514248933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:28,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:28,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:28,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:28,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44986 deadline: 1732514248933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:28,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:28,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:28,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:28,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:28,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:29,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:29,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514249052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:29,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:29,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:29,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:29,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44986 deadline: 1732514249052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:29,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:29,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:29,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:29,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:29,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:29,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514249052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:29,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:29,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-25T05:56:29,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:29,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:29,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:29,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:29,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:29,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:29,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-25T05:56:29,088 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/8b74f235df864c4daa15470f9cb725ab as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/8b74f235df864c4daa15470f9cb725ab 2024-11-25T05:56:29,092 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef2460dfb8241e76bc050e18474be77d/B of ef2460dfb8241e76bc050e18474be77d into 8b74f235df864c4daa15470f9cb725ab(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:56:29,092 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef2460dfb8241e76bc050e18474be77d: 2024-11-25T05:56:29,092 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., storeName=ef2460dfb8241e76bc050e18474be77d/B, priority=13, startTime=1732514188668; duration=0sec 2024-11-25T05:56:29,092 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:56:29,092 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef2460dfb8241e76bc050e18474be77d:B 2024-11-25T05:56:29,092 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:56:29,093 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:56:29,093 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): ef2460dfb8241e76bc050e18474be77d/C is initiating minor compaction (all files) 2024-11-25T05:56:29,093 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef2460dfb8241e76bc050e18474be77d/C in TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:29,093 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/5b87bc3d74ec409e9fe5d4f2a4bc195b, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/add9b508983046dbace60a552116a048, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/698c56d0f6594c7c8eb3cde6a547678f] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp, totalSize=35.2 K 2024-11-25T05:56:29,094 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b87bc3d74ec409e9fe5d4f2a4bc195b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732514184599 2024-11-25T05:56:29,094 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef2460dfb8241e76bc050e18474be77d#A#compaction#320 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:56:29,094 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting add9b508983046dbace60a552116a048, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732514184691 2024-11-25T05:56:29,094 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 698c56d0f6594c7c8eb3cde6a547678f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732514185850 2024-11-25T05:56:29,094 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/f9290140b32840fd9ea92287fdde5770 is 175, key is test_row_0/A:col10/1732514187004/Put/seqid=0 2024-11-25T05:56:29,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742198_1374 (size=31058) 2024-11-25T05:56:29,101 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef2460dfb8241e76bc050e18474be77d#C#compaction#323 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:56:29,101 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/4d5a7c71b9654d67ac708492a40095a5 is 50, key is test_row_0/C:col10/1732514187004/Put/seqid=0 2024-11-25T05:56:29,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742199_1375 (size=12104) 2024-11-25T05:56:29,115 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/4d5a7c71b9654d67ac708492a40095a5 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/4d5a7c71b9654d67ac708492a40095a5 2024-11-25T05:56:29,119 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef2460dfb8241e76bc050e18474be77d/C of ef2460dfb8241e76bc050e18474be77d into 4d5a7c71b9654d67ac708492a40095a5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:56:29,119 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef2460dfb8241e76bc050e18474be77d: 2024-11-25T05:56:29,119 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., storeName=ef2460dfb8241e76bc050e18474be77d/C, priority=13, startTime=1732514188668; duration=0sec 2024-11-25T05:56:29,119 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:29,119 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef2460dfb8241e76bc050e18474be77d:C 2024-11-25T05:56:29,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-25T05:56:29,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:29,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:29,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:29,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:29,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:29,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44954 deadline: 1732514249222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:29,233 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_1' in ef2460dfb8241e76bc050e18474be77d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:56:29,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:29,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:29,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:29,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44934 deadline: 1732514249232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:29,235 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_1' in ef2460dfb8241e76bc050e18474be77d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:56:29,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:29,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:29,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:29,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514249262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:29,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:29,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:29,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:29,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44986 deadline: 1732514249262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:29,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:29,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:29,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:29,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514249262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:29,320 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/979e8d987041446c96c1222d08e759ae 2024-11-25T05:56:29,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/92e5f727b6c7409da4fe05d8c4b4e108 is 50, key is test_row_0/B:col10/1732514187053/Put/seqid=0 2024-11-25T05:56:29,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742200_1376 (size=12001) 2024-11-25T05:56:29,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-25T05:56:29,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:29,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:29,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:29,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:29,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:29,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:29,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-25T05:56:29,502 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/f9290140b32840fd9ea92287fdde5770 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/f9290140b32840fd9ea92287fdde5770 2024-11-25T05:56:29,505 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef2460dfb8241e76bc050e18474be77d/A of ef2460dfb8241e76bc050e18474be77d into f9290140b32840fd9ea92287fdde5770(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:56:29,505 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef2460dfb8241e76bc050e18474be77d: 2024-11-25T05:56:29,505 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., storeName=ef2460dfb8241e76bc050e18474be77d/A, priority=13, startTime=1732514188668; duration=0sec 2024-11-25T05:56:29,505 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:29,506 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef2460dfb8241e76bc050e18474be77d:A 2024-11-25T05:56:29,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-25T05:56:29,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:29,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:29,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:29,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:29,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:29,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:29,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514249582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:29,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:29,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:29,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:29,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:29,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44986 deadline: 1732514249582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:29,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:29,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:29,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:29,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:29,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514249582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:29,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:29,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:29,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:29,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-25T05:56:29,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:29,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:29,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:29,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:29,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:29,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:29,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/92e5f727b6c7409da4fe05d8c4b4e108 2024-11-25T05:56:29,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/835e4a2fb0594e2eb1d045a220946e75 is 50, key is test_row_0/C:col10/1732514187053/Put/seqid=0 2024-11-25T05:56:29,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742201_1377 (size=12001) 2024-11-25T05:56:29,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-25T05:56:29,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:29,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:29,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:29,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:29,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:29,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:29,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-25T05:56:29,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-25T05:56:29,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:29,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:29,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:29,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:29,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:29,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:30,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:30,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514250102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:30,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:30,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:30,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:30,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:30,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44986 deadline: 1732514250102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:30,113 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:30,113 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:30,113 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:30,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:30,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514250102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:30,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:30,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:30,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:30,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-25T05:56:30,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:30,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:30,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:30,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:30,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:30,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:30,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/835e4a2fb0594e2eb1d045a220946e75 2024-11-25T05:56:30,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/979e8d987041446c96c1222d08e759ae as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/979e8d987041446c96c1222d08e759ae 2024-11-25T05:56:30,149 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/979e8d987041446c96c1222d08e759ae, entries=200, sequenceid=81, filesize=38.6 K 2024-11-25T05:56:30,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/92e5f727b6c7409da4fe05d8c4b4e108 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/92e5f727b6c7409da4fe05d8c4b4e108 2024-11-25T05:56:30,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/92e5f727b6c7409da4fe05d8c4b4e108, entries=150, sequenceid=81, filesize=11.7 K 2024-11-25T05:56:30,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/835e4a2fb0594e2eb1d045a220946e75 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/835e4a2fb0594e2eb1d045a220946e75 2024-11-25T05:56:30,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/835e4a2fb0594e2eb1d045a220946e75, entries=150, sequenceid=81, filesize=11.7 K 2024-11-25T05:56:30,157 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for ef2460dfb8241e76bc050e18474be77d in 1260ms, sequenceid=81, compaction requested=false 2024-11-25T05:56:30,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ef2460dfb8241e76bc050e18474be77d: 2024-11-25T05:56:30,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-25T05:56:30,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:30,284 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing ef2460dfb8241e76bc050e18474be77d 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-25T05:56:30,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=A 2024-11-25T05:56:30,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:30,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=B 2024-11-25T05:56:30,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:30,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=C 2024-11-25T05:56:30,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:30,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125bd89193fb0704af3b0268e6b28d8be24_ef2460dfb8241e76bc050e18474be77d is 50, key is test_row_0/A:col10/1732514188920/Put/seqid=0 2024-11-25T05:56:30,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742202_1378 (size=12154) 2024-11-25T05:56:30,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:56:30,297 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125bd89193fb0704af3b0268e6b28d8be24_ef2460dfb8241e76bc050e18474be77d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125bd89193fb0704af3b0268e6b28d8be24_ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:30,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/85e0727b5ff24c3ea8b7d1e1c3f3c671, store: [table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:30,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/85e0727b5ff24c3ea8b7d1e1c3f3c671 is 175, key is test_row_0/A:col10/1732514188920/Put/seqid=0 2024-11-25T05:56:30,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742203_1379 (size=30955) 2024-11-25T05:56:30,704 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/85e0727b5ff24c3ea8b7d1e1c3f3c671 2024-11-25T05:56:30,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/070c5d7fae994adb936dc8f4ea584169 is 50, key is test_row_0/B:col10/1732514188920/Put/seqid=0 2024-11-25T05:56:30,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742204_1380 (size=12001) 2024-11-25T05:56:30,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-25T05:56:31,114 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/070c5d7fae994adb936dc8f4ea584169 2024-11-25T05:56:31,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/c2d074166f714f058ad41c4a6c39f5ad is 50, key is test_row_0/C:col10/1732514188920/Put/seqid=0 2024-11-25T05:56:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:31,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:31,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742205_1381 (size=12001) 2024-11-25T05:56:31,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:31,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44986 deadline: 1732514251168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:31,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:31,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:31,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:31,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732514191155, max timestamp=1732514191174Num uniques 859; 2024-11-25T05:56:31,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732514191155, max timestamp=1732514191174Num uniques 859; 2024-11-25T05:56:31,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732514191155, max timestamp=1732514191174Num uniques 859; 2024-11-25T05:56:31,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-25T05:56:31,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-25T05:56:31,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-25T05:56:31,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:31,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514251243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:31,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:31,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:31,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:31,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:31,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514251243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:31,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:31,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:31,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:31,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:31,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44934 deadline: 1732514251252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:31,257 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_1' in ef2460dfb8241e76bc050e18474be77d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:56:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:31,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:31,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44954 deadline: 1732514251262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:31,270 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_1' in ef2460dfb8241e76bc050e18474be77d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:56:31,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:31,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:31,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:31,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:31,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44986 deadline: 1732514251282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:31,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:31,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:31,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:31,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:31,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514251362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:31,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:31,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:31,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:31,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:31,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514251362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:31,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:31,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:31,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:31,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44986 deadline: 1732514251501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:31,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:31,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:31,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:31,529 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/c2d074166f714f058ad41c4a6c39f5ad 2024-11-25T05:56:31,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/85e0727b5ff24c3ea8b7d1e1c3f3c671 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/85e0727b5ff24c3ea8b7d1e1c3f3c671 2024-11-25T05:56:31,535 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/85e0727b5ff24c3ea8b7d1e1c3f3c671, entries=150, sequenceid=95, filesize=30.2 K 2024-11-25T05:56:31,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/070c5d7fae994adb936dc8f4ea584169 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/070c5d7fae994adb936dc8f4ea584169 2024-11-25T05:56:31,539 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/070c5d7fae994adb936dc8f4ea584169, entries=150, sequenceid=95, filesize=11.7 K 2024-11-25T05:56:31,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/c2d074166f714f058ad41c4a6c39f5ad as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/c2d074166f714f058ad41c4a6c39f5ad 2024-11-25T05:56:31,542 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/c2d074166f714f058ad41c4a6c39f5ad, entries=150, sequenceid=95, filesize=11.7 K 2024-11-25T05:56:31,543 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ef2460dfb8241e76bc050e18474be77d in 1259ms, sequenceid=95, compaction requested=true 2024-11-25T05:56:31,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for ef2460dfb8241e76bc050e18474be77d: 2024-11-25T05:56:31,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:31,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-25T05:56:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-25T05:56:31,550 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-25T05:56:31,550 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7930 sec 2024-11-25T05:56:31,552 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.7980 sec 2024-11-25T05:56:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:31,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ef2460dfb8241e76bc050e18474be77d 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-25T05:56:31,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=A 2024-11-25T05:56:31,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:56:31,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=B 2024-11-25T05:56:31,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:56:31,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=C 2024-11-25T05:56:31,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:56:31,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411255d58c8e3efe040f2acc3211cc10ecb4d_ef2460dfb8241e76bc050e18474be77d is 50, key is test_row_0/A:col10/1732514191239/Put/seqid=0 2024-11-25T05:56:31,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742206_1382 (size=14644) 2024-11-25T05:56:31,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514251607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:31,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:31,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:31,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:31,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514251611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:31,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:31,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:31,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:31,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:31,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514251721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:31,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:31,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:31,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:31,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:31,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514251732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:31,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:31,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:31,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:31,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:31,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44986 deadline: 1732514251821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:31,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:31,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:31,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:31,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:31,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514251941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:31,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:31,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:31,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:31,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:31,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514251952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:31,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:31,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:31,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:31,990 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:56:31,993 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411255d58c8e3efe040f2acc3211cc10ecb4d_ef2460dfb8241e76bc050e18474be77d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411255d58c8e3efe040f2acc3211cc10ecb4d_ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:31,993 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/4d4b0bbf03bc46b59386153f52f35ac7, store: [table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:31,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/4d4b0bbf03bc46b59386153f52f35ac7 is 175, key is test_row_0/A:col10/1732514191239/Put/seqid=0 2024-11-25T05:56:31,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742207_1383 (size=39599) 2024-11-25T05:56:32,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:32,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514252252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:32,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:32,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:32,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:32,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:32,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514252262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:32,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:32,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:32,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:32,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:32,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44986 deadline: 1732514252342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:32,349 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:32,349 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:32,349 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:32,397 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/4d4b0bbf03bc46b59386153f52f35ac7 2024-11-25T05:56:32,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/14fa66276ef448368e5a3c15684c5fcb is 50, key is test_row_0/B:col10/1732514191239/Put/seqid=0 2024-11-25T05:56:32,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742208_1384 (size=12051) 2024-11-25T05:56:32,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:32,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514252772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:32,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:32,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:32,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:32,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:32,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514252782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:32,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:32,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:32,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:32,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/14fa66276ef448368e5a3c15684c5fcb 2024-11-25T05:56:32,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/250c260f8cb64b1986d19a8c001453d7 is 50, key is test_row_0/C:col10/1732514191239/Put/seqid=0 2024-11-25T05:56:32,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742209_1385 (size=12051) 2024-11-25T05:56:32,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-25T05:56:32,912 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:56:32,912 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:56:32,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-25T05:56:32,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-25T05:56:32,914 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:56:32,914 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:56:32,914 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:56:33,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-25T05:56:33,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T05:56:33,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:33,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:33,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:33,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:33,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:33,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:33,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/250c260f8cb64b1986d19a8c001453d7 2024-11-25T05:56:33,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/4d4b0bbf03bc46b59386153f52f35ac7 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/4d4b0bbf03bc46b59386153f52f35ac7 2024-11-25T05:56:33,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T05:56:33,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:33,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:33,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:33,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:33,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:33,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:33,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/4d4b0bbf03bc46b59386153f52f35ac7, entries=200, sequenceid=130, filesize=38.7 K 2024-11-25T05:56:33,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/14fa66276ef448368e5a3c15684c5fcb as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/14fa66276ef448368e5a3c15684c5fcb 2024-11-25T05:56:33,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/14fa66276ef448368e5a3c15684c5fcb, entries=150, sequenceid=130, filesize=11.8 K 2024-11-25T05:56:33,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/250c260f8cb64b1986d19a8c001453d7 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/250c260f8cb64b1986d19a8c001453d7 2024-11-25T05:56:33,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-25T05:56:33,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/250c260f8cb64b1986d19a8c001453d7, entries=150, sequenceid=130, filesize=11.8 K 2024-11-25T05:56:33,234 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for ef2460dfb8241e76bc050e18474be77d in 1655ms, sequenceid=130, compaction requested=true 2024-11-25T05:56:33,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ef2460dfb8241e76bc050e18474be77d: 2024-11-25T05:56:33,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef2460dfb8241e76bc050e18474be77d:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:56:33,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:33,234 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T05:56:33,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef2460dfb8241e76bc050e18474be77d:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:56:33,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:33,234 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T05:56:33,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef2460dfb8241e76bc050e18474be77d:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:56:33,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:56:33,235 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48157 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T05:56:33,235 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141161 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T05:56:33,235 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): ef2460dfb8241e76bc050e18474be77d/A is initiating minor compaction (all files) 2024-11-25T05:56:33,235 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): ef2460dfb8241e76bc050e18474be77d/B is initiating minor compaction (all files) 2024-11-25T05:56:33,235 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef2460dfb8241e76bc050e18474be77d/A in TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:33,235 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef2460dfb8241e76bc050e18474be77d/B in TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:33,236 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/8b74f235df864c4daa15470f9cb725ab, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/92e5f727b6c7409da4fe05d8c4b4e108, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/070c5d7fae994adb936dc8f4ea584169, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/14fa66276ef448368e5a3c15684c5fcb] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp, totalSize=47.0 K 2024-11-25T05:56:33,236 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/f9290140b32840fd9ea92287fdde5770, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/979e8d987041446c96c1222d08e759ae, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/85e0727b5ff24c3ea8b7d1e1c3f3c671, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/4d4b0bbf03bc46b59386153f52f35ac7] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp, totalSize=137.9 K 2024-11-25T05:56:33,236 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:33,236 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. files: [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/f9290140b32840fd9ea92287fdde5770, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/979e8d987041446c96c1222d08e759ae, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/85e0727b5ff24c3ea8b7d1e1c3f3c671, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/4d4b0bbf03bc46b59386153f52f35ac7] 2024-11-25T05:56:33,236 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b74f235df864c4daa15470f9cb725ab, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732514185850 2024-11-25T05:56:33,236 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 92e5f727b6c7409da4fe05d8c4b4e108, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732514187053 2024-11-25T05:56:33,236 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 070c5d7fae994adb936dc8f4ea584169, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732514188913 2024-11-25T05:56:33,237 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9290140b32840fd9ea92287fdde5770, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732514185850 2024-11-25T05:56:33,237 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 14fa66276ef448368e5a3c15684c5fcb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732514191200 2024-11-25T05:56:33,237 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 979e8d987041446c96c1222d08e759ae, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732514187052 2024-11-25T05:56:33,238 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85e0727b5ff24c3ea8b7d1e1c3f3c671, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732514188913 2024-11-25T05:56:33,238 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d4b0bbf03bc46b59386153f52f35ac7, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732514191200 2024-11-25T05:56:33,246 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef2460dfb8241e76bc050e18474be77d#B#compaction#332 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:56:33,246 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/e8e27d4174684e86b0729544f21c1cab is 50, key is test_row_0/B:col10/1732514191239/Put/seqid=0 2024-11-25T05:56:33,248 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:33,250 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125eff3a9bcfa0a443fa090058aabfae8e9_ef2460dfb8241e76bc050e18474be77d store=[table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:33,253 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125eff3a9bcfa0a443fa090058aabfae8e9_ef2460dfb8241e76bc050e18474be77d, store=[table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:33,254 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125eff3a9bcfa0a443fa090058aabfae8e9_ef2460dfb8241e76bc050e18474be77d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:33,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742210_1386 (size=12291) 2024-11-25T05:56:33,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742211_1387 (size=4469) 2024-11-25T05:56:33,281 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/e8e27d4174684e86b0729544f21c1cab as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/e8e27d4174684e86b0729544f21c1cab 2024-11-25T05:56:33,283 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef2460dfb8241e76bc050e18474be77d#A#compaction#333 average throughput is 0.70 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:56:33,283 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/969e3b9557d94a2291edf3e96bd189ca is 175, key is test_row_0/A:col10/1732514191239/Put/seqid=0 2024-11-25T05:56:33,289 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ef2460dfb8241e76bc050e18474be77d/B of ef2460dfb8241e76bc050e18474be77d into e8e27d4174684e86b0729544f21c1cab(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:56:33,289 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef2460dfb8241e76bc050e18474be77d: 2024-11-25T05:56:33,289 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., storeName=ef2460dfb8241e76bc050e18474be77d/B, priority=12, startTime=1732514193234; duration=0sec 2024-11-25T05:56:33,290 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:56:33,290 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef2460dfb8241e76bc050e18474be77d:B 2024-11-25T05:56:33,290 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T05:56:33,291 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48157 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T05:56:33,291 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): ef2460dfb8241e76bc050e18474be77d/C is initiating minor compaction (all files) 2024-11-25T05:56:33,292 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef2460dfb8241e76bc050e18474be77d/C in TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:33,292 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/4d5a7c71b9654d67ac708492a40095a5, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/835e4a2fb0594e2eb1d045a220946e75, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/c2d074166f714f058ad41c4a6c39f5ad, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/250c260f8cb64b1986d19a8c001453d7] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp, totalSize=47.0 K 2024-11-25T05:56:33,292 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d5a7c71b9654d67ac708492a40095a5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732514185850 2024-11-25T05:56:33,293 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 835e4a2fb0594e2eb1d045a220946e75, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732514187053 2024-11-25T05:56:33,294 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting c2d074166f714f058ad41c4a6c39f5ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732514188913 2024-11-25T05:56:33,296 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 250c260f8cb64b1986d19a8c001453d7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732514191200 2024-11-25T05:56:33,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742212_1388 (size=31245) 2024-11-25T05:56:33,311 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/969e3b9557d94a2291edf3e96bd189ca as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/969e3b9557d94a2291edf3e96bd189ca 2024-11-25T05:56:33,316 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ef2460dfb8241e76bc050e18474be77d/A of ef2460dfb8241e76bc050e18474be77d into 969e3b9557d94a2291edf3e96bd189ca(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:56:33,317 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef2460dfb8241e76bc050e18474be77d: 2024-11-25T05:56:33,317 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., storeName=ef2460dfb8241e76bc050e18474be77d/A, priority=12, startTime=1732514193234; duration=0sec 2024-11-25T05:56:33,317 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:33,317 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef2460dfb8241e76bc050e18474be77d:A 2024-11-25T05:56:33,326 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef2460dfb8241e76bc050e18474be77d#C#compaction#334 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:56:33,327 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/50db4314e1ef4a74b84b4afb78fd9822 is 50, key is test_row_0/C:col10/1732514191239/Put/seqid=0 2024-11-25T05:56:33,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742213_1389 (size=12291) 2024-11-25T05:56:33,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:33,370 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ef2460dfb8241e76bc050e18474be77d 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-25T05:56:33,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=A 2024-11-25T05:56:33,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:33,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=B 2024-11-25T05:56:33,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:33,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=C 2024-11-25T05:56:33,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:33,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T05:56:33,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:33,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:33,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:33,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:33,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:33,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:33,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125791259ecd65a48bd8efe8969f38ed277_ef2460dfb8241e76bc050e18474be77d is 50, key is test_row_0/A:col10/1732514191606/Put/seqid=0 2024-11-25T05:56:33,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742214_1390 (size=14794) 2024-11-25T05:56:33,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732514193459, max timestamp=1732514193484Num uniques 859; 2024-11-25T05:56:33,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732514193459, max timestamp=1732514193484Num uniques 859; 2024-11-25T05:56:33,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T05:56:33,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T05:56:33,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732514193459, max timestamp=1732514193484Num uniques 859; 2024-11-25T05:56:33,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-25T05:56:33,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T05:56:33,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:33,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:33,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:33,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:33,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:33,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:33,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-25T05:56:33,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:33,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44986 deadline: 1732514253607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:33,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:33,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:33,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:33,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T05:56:33,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:33,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:33,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:33,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:33,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:33,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:33,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:33,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44986 deadline: 1732514253722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:33,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:33,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:33,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:33,741 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/50db4314e1ef4a74b84b4afb78fd9822 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/50db4314e1ef4a74b84b4afb78fd9822 2024-11-25T05:56:33,746 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ef2460dfb8241e76bc050e18474be77d/C of ef2460dfb8241e76bc050e18474be77d into 50db4314e1ef4a74b84b4afb78fd9822(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:56:33,746 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef2460dfb8241e76bc050e18474be77d: 2024-11-25T05:56:33,746 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., storeName=ef2460dfb8241e76bc050e18474be77d/C, priority=12, startTime=1732514193234; duration=0sec 2024-11-25T05:56:33,747 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:33,747 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef2460dfb8241e76bc050e18474be77d:C 2024-11-25T05:56:33,779 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:56:33,783 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125791259ecd65a48bd8efe8969f38ed277_ef2460dfb8241e76bc050e18474be77d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125791259ecd65a48bd8efe8969f38ed277_ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:33,784 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/04ee38a92d83411ea8c62b9617672591, store: [table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:33,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/04ee38a92d83411ea8c62b9617672591 is 175, key is test_row_0/A:col10/1732514191606/Put/seqid=0 2024-11-25T05:56:33,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742215_1391 (size=39749) 2024-11-25T05:56:33,790 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=143, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/04ee38a92d83411ea8c62b9617672591 2024-11-25T05:56:33,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/4e49c0474658488ba552371b183902ef is 50, key is test_row_0/B:col10/1732514191606/Put/seqid=0 2024-11-25T05:56:33,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:33,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514253792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:33,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:33,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514253792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:33,804 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_0' in ef2460dfb8241e76bc050e18474be77d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:56:33,804 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_2' in ef2460dfb8241e76bc050e18474be77d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:56:33,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:33,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:33,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:33,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:33,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:33,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:33,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742216_1392 (size=12151) 2024-11-25T05:56:33,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/4e49c0474658488ba552371b183902ef 2024-11-25T05:56:33,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T05:56:33,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:33,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:33,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:33,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:33,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:33,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:33,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/0b12471b2abe45b7a3ba9bd1b92cf5f7 is 50, key is test_row_0/C:col10/1732514191606/Put/seqid=0 2024-11-25T05:56:33,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742217_1393 (size=12151) 2024-11-25T05:56:33,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:33,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44986 deadline: 1732514253942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:33,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:33,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:33,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:33,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T05:56:33,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:33,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:33,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:33,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:33,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:33,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:34,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-25T05:56:34,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T05:56:34,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:34,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:34,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:34,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:34,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:34,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:34,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/0b12471b2abe45b7a3ba9bd1b92cf5f7 2024-11-25T05:56:34,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/04ee38a92d83411ea8c62b9617672591 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/04ee38a92d83411ea8c62b9617672591 2024-11-25T05:56:34,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/04ee38a92d83411ea8c62b9617672591, entries=200, sequenceid=143, filesize=38.8 K 2024-11-25T05:56:34,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/4e49c0474658488ba552371b183902ef as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/4e49c0474658488ba552371b183902ef 2024-11-25T05:56:34,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/4e49c0474658488ba552371b183902ef, entries=150, sequenceid=143, filesize=11.9 K 2024-11-25T05:56:34,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/0b12471b2abe45b7a3ba9bd1b92cf5f7 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/0b12471b2abe45b7a3ba9bd1b92cf5f7 2024-11-25T05:56:34,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/0b12471b2abe45b7a3ba9bd1b92cf5f7, entries=150, sequenceid=143, filesize=11.9 K 2024-11-25T05:56:34,253 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ef2460dfb8241e76bc050e18474be77d in 883ms, sequenceid=143, compaction requested=false 2024-11-25T05:56:34,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ef2460dfb8241e76bc050e18474be77d: 2024-11-25T05:56:34,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:34,267 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ef2460dfb8241e76bc050e18474be77d 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-25T05:56:34,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=A 2024-11-25T05:56:34,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:56:34,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=B 2024-11-25T05:56:34,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:56:34,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=C 2024-11-25T05:56:34,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:56:34,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125d2d65cd494414891834e9e6689527e60_ef2460dfb8241e76bc050e18474be77d is 50, key is test_row_0/A:col10/1732514194265/Put/seqid=0 2024-11-25T05:56:34,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742218_1394 (size=14794) 2024-11-25T05:56:34,277 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:56:34,281 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125d2d65cd494414891834e9e6689527e60_ef2460dfb8241e76bc050e18474be77d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d2d65cd494414891834e9e6689527e60_ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:34,282 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/a0aad8fc67e1434cbe53c44cd1785221, store: [table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:34,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/a0aad8fc67e1434cbe53c44cd1785221 is 175, key is test_row_0/A:col10/1732514194265/Put/seqid=0 2024-11-25T05:56:34,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742219_1395 (size=39749) 2024-11-25T05:56:34,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T05:56:34,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:34,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:34,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:34,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:34,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:34,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:34,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:34,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44986 deadline: 1732514254317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:34,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:34,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:34,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:34,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:34,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44986 deadline: 1732514254432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:34,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T05:56:34,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:34,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:34,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:34,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:34,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:34,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:34,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:34,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:34,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:34,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T05:56:34,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:34,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:34,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:34,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:34,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:34,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:34,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:34,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44986 deadline: 1732514254652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:34,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:34,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:34,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:34,686 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=181, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/a0aad8fc67e1434cbe53c44cd1785221 2024-11-25T05:56:34,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/385cbe658682415db4c1e10869caae6b is 50, key is test_row_0/B:col10/1732514194265/Put/seqid=0 2024-11-25T05:56:34,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742220_1396 (size=12151) 2024-11-25T05:56:34,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T05:56:34,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:34,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:34,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:34,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:34,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:34,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:34,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T05:56:34,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:34,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:34,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:34,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:34,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:34,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:34,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:34,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44986 deadline: 1732514254972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:34,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:34,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:34,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:35,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T05:56:35,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:35,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:35,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:35,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:35,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:35,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:35,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-25T05:56:35,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/385cbe658682415db4c1e10869caae6b 2024-11-25T05:56:35,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/5725dd2bdd9e4fcfa14ebd47f816ef38 is 50, key is test_row_0/C:col10/1732514194265/Put/seqid=0 2024-11-25T05:56:35,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742221_1397 (size=12151) 2024-11-25T05:56:35,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T05:56:35,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:35,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:35,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:35,206 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:35,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:35,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:35,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:35,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44934 deadline: 1732514255292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:35,295 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_1' in ef2460dfb8241e76bc050e18474be77d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:56:35,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:35,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:35,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:35,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:35,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44954 deadline: 1732514255312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:35,322 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_1' in ef2460dfb8241e76bc050e18474be77d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8260 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:56:35,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:35,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:35,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:35,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T05:56:35,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:35,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:35,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:35,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:35,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:35,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:35,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:35,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44986 deadline: 1732514255492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:35,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:35,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:35,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:35,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/5725dd2bdd9e4fcfa14ebd47f816ef38 2024-11-25T05:56:35,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/a0aad8fc67e1434cbe53c44cd1785221 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/a0aad8fc67e1434cbe53c44cd1785221 2024-11-25T05:56:35,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T05:56:35,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:35,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:35,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:35,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:35,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:35,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:35,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/a0aad8fc67e1434cbe53c44cd1785221, entries=200, sequenceid=181, filesize=38.8 K 2024-11-25T05:56:35,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/385cbe658682415db4c1e10869caae6b as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/385cbe658682415db4c1e10869caae6b 2024-11-25T05:56:35,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/385cbe658682415db4c1e10869caae6b, entries=150, sequenceid=181, filesize=11.9 K 2024-11-25T05:56:35,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/5725dd2bdd9e4fcfa14ebd47f816ef38 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/5725dd2bdd9e4fcfa14ebd47f816ef38 2024-11-25T05:56:35,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/5725dd2bdd9e4fcfa14ebd47f816ef38, entries=150, sequenceid=181, filesize=11.9 K 2024-11-25T05:56:35,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for ef2460dfb8241e76bc050e18474be77d in 1258ms, sequenceid=181, compaction requested=true 2024-11-25T05:56:35,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ef2460dfb8241e76bc050e18474be77d: 2024-11-25T05:56:35,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef2460dfb8241e76bc050e18474be77d:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:56:35,524 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:56:35,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:35,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef2460dfb8241e76bc050e18474be77d:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:56:35,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:35,524 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:56:35,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef2460dfb8241e76bc050e18474be77d:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:56:35,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:56:35,525 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110743 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:56:35,525 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36593 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:56:35,525 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): ef2460dfb8241e76bc050e18474be77d/A is initiating minor compaction (all files) 2024-11-25T05:56:35,525 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): ef2460dfb8241e76bc050e18474be77d/B is initiating minor compaction (all files) 2024-11-25T05:56:35,525 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef2460dfb8241e76bc050e18474be77d/A in TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:35,525 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef2460dfb8241e76bc050e18474be77d/B in TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:35,525 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/e8e27d4174684e86b0729544f21c1cab, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/4e49c0474658488ba552371b183902ef, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/385cbe658682415db4c1e10869caae6b] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp, totalSize=35.7 K 2024-11-25T05:56:35,525 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/969e3b9557d94a2291edf3e96bd189ca, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/04ee38a92d83411ea8c62b9617672591, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/a0aad8fc67e1434cbe53c44cd1785221] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp, totalSize=108.1 K 2024-11-25T05:56:35,525 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:35,525 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. files: [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/969e3b9557d94a2291edf3e96bd189ca, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/04ee38a92d83411ea8c62b9617672591, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/a0aad8fc67e1434cbe53c44cd1785221] 2024-11-25T05:56:35,525 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting e8e27d4174684e86b0729544f21c1cab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732514191200 2024-11-25T05:56:35,525 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 969e3b9557d94a2291edf3e96bd189ca, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732514191200 2024-11-25T05:56:35,525 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e49c0474658488ba552371b183902ef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732514191606 2024-11-25T05:56:35,526 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04ee38a92d83411ea8c62b9617672591, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732514191592 2024-11-25T05:56:35,526 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 385cbe658682415db4c1e10869caae6b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732514193577 2024-11-25T05:56:35,526 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0aad8fc67e1434cbe53c44cd1785221, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732514193577 2024-11-25T05:56:35,530 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:35,531 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef2460dfb8241e76bc050e18474be77d#B#compaction#341 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:56:35,532 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/5ab476f341f54c188b704aa937b397c4 is 50, key is test_row_0/B:col10/1732514194265/Put/seqid=0 2024-11-25T05:56:35,537 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411256b5c632c74074f9fa77e4a4e90c0be75_ef2460dfb8241e76bc050e18474be77d store=[table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:35,539 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411256b5c632c74074f9fa77e4a4e90c0be75_ef2460dfb8241e76bc050e18474be77d, store=[table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:35,539 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411256b5c632c74074f9fa77e4a4e90c0be75_ef2460dfb8241e76bc050e18474be77d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:35,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742222_1398 (size=12493) 2024-11-25T05:56:35,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742223_1399 (size=4469) 2024-11-25T05:56:35,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T05:56:35,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:35,665 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing ef2460dfb8241e76bc050e18474be77d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-25T05:56:35,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=A 2024-11-25T05:56:35,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:35,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=B 2024-11-25T05:56:35,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:35,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=C 2024-11-25T05:56:35,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:35,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125bb8d8231f030497d955e5690227eb854_ef2460dfb8241e76bc050e18474be77d is 50, key is test_row_0/A:col10/1732514194316/Put/seqid=0 2024-11-25T05:56:35,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742224_1400 (size=12304) 2024-11-25T05:56:35,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:35,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:35,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732514195873, max timestamp=1732514195909Num uniques 859; 2024-11-25T05:56:35,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732514195873, max timestamp=1732514195909Num uniques 859; 2024-11-25T05:56:35,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732514195873, max timestamp=1732514195909Num uniques 859; 2024-11-25T05:56:35,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T05:56:35,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T05:56:35,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T05:56:35,946 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/5ab476f341f54c188b704aa937b397c4 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/5ab476f341f54c188b704aa937b397c4 2024-11-25T05:56:35,949 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef2460dfb8241e76bc050e18474be77d/B of ef2460dfb8241e76bc050e18474be77d into 5ab476f341f54c188b704aa937b397c4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:56:35,950 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef2460dfb8241e76bc050e18474be77d: 2024-11-25T05:56:35,950 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., storeName=ef2460dfb8241e76bc050e18474be77d/B, priority=13, startTime=1732514195524; duration=0sec 2024-11-25T05:56:35,950 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:56:35,950 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef2460dfb8241e76bc050e18474be77d:B 2024-11-25T05:56:35,950 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:56:35,951 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36593 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:56:35,951 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): ef2460dfb8241e76bc050e18474be77d/C is initiating minor compaction (all files) 2024-11-25T05:56:35,951 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef2460dfb8241e76bc050e18474be77d/C in TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:35,951 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/50db4314e1ef4a74b84b4afb78fd9822, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/0b12471b2abe45b7a3ba9bd1b92cf5f7, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/5725dd2bdd9e4fcfa14ebd47f816ef38] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp, totalSize=35.7 K 2024-11-25T05:56:35,951 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 50db4314e1ef4a74b84b4afb78fd9822, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732514191200 2024-11-25T05:56:35,952 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef2460dfb8241e76bc050e18474be77d#A#compaction#342 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:56:35,952 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/2db260a0c5c7406db7d38ce725e0bc2b is 175, key is test_row_0/A:col10/1732514194265/Put/seqid=0 2024-11-25T05:56:35,952 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b12471b2abe45b7a3ba9bd1b92cf5f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732514191606 2024-11-25T05:56:35,953 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 5725dd2bdd9e4fcfa14ebd47f816ef38, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732514193577 2024-11-25T05:56:35,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742225_1401 (size=31447) 2024-11-25T05:56:35,964 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef2460dfb8241e76bc050e18474be77d#C#compaction#344 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:56:35,965 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/c89b61141f1b4e41b5d01a198b516938 is 50, key is test_row_0/C:col10/1732514194265/Put/seqid=0 2024-11-25T05:56:35,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742226_1402 (size=12493) 2024-11-25T05:56:35,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:35,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514255985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:35,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:35,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:35,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:36,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:36,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514255992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:36,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:36,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:36,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:36,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:56:36,078 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125bb8d8231f030497d955e5690227eb854_ef2460dfb8241e76bc050e18474be77d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125bb8d8231f030497d955e5690227eb854_ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:36,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/e69ff81d5c1a4757b258a6e15cee29f4, store: [table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:36,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/e69ff81d5c1a4757b258a6e15cee29f4 is 175, key is test_row_0/A:col10/1732514194316/Put/seqid=0 2024-11-25T05:56:36,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742227_1403 (size=31105) 2024-11-25T05:56:36,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:36,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514256102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:36,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:36,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:36,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:36,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514256112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:36,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:36,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:36,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:36,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:36,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514256322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:36,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:36,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514256322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:36,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:36,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:36,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:36,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:36,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:36,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:36,361 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/2db260a0c5c7406db7d38ce725e0bc2b as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/2db260a0c5c7406db7d38ce725e0bc2b 2024-11-25T05:56:36,365 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef2460dfb8241e76bc050e18474be77d/A of ef2460dfb8241e76bc050e18474be77d into 2db260a0c5c7406db7d38ce725e0bc2b(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:56:36,365 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef2460dfb8241e76bc050e18474be77d: 2024-11-25T05:56:36,365 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., storeName=ef2460dfb8241e76bc050e18474be77d/A, priority=13, startTime=1732514195524; duration=0sec 2024-11-25T05:56:36,365 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:36,365 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef2460dfb8241e76bc050e18474be77d:A 2024-11-25T05:56:36,373 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/c89b61141f1b4e41b5d01a198b516938 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/c89b61141f1b4e41b5d01a198b516938 2024-11-25T05:56:36,377 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef2460dfb8241e76bc050e18474be77d/C of ef2460dfb8241e76bc050e18474be77d into c89b61141f1b4e41b5d01a198b516938(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:56:36,377 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef2460dfb8241e76bc050e18474be77d: 2024-11-25T05:56:36,377 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., storeName=ef2460dfb8241e76bc050e18474be77d/C, priority=13, startTime=1732514195524; duration=0sec 2024-11-25T05:56:36,377 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:36,377 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef2460dfb8241e76bc050e18474be77d:C 2024-11-25T05:56:36,483 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=191, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/e69ff81d5c1a4757b258a6e15cee29f4 2024-11-25T05:56:36,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/e53f3fd6372c42b79d85f4ce8eadb0d9 is 50, key is test_row_0/B:col10/1732514194316/Put/seqid=0 2024-11-25T05:56:36,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742228_1404 (size=12151) 2024-11-25T05:56:36,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:36,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44986 deadline: 1732514256522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:36,529 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_2' in ef2460dfb8241e76bc050e18474be77d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:56:36,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:36,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:36,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:36,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:36,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514256642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:36,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:36,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:36,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514256642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:36,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:36,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:36,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:36,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:36,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:36,894 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/e53f3fd6372c42b79d85f4ce8eadb0d9 2024-11-25T05:56:36,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/2b031c403164448daeb3f61a048ca7ac is 50, key is test_row_0/C:col10/1732514194316/Put/seqid=0 2024-11-25T05:56:36,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742229_1405 (size=12151) 2024-11-25T05:56:37,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-25T05:56:37,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:37,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514257152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:37,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:37,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:37,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:37,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:37,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514257162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:37,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:37,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:37,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:37,308 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/2b031c403164448daeb3f61a048ca7ac 2024-11-25T05:56:37,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/e69ff81d5c1a4757b258a6e15cee29f4 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/e69ff81d5c1a4757b258a6e15cee29f4 2024-11-25T05:56:37,315 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/e69ff81d5c1a4757b258a6e15cee29f4, entries=150, sequenceid=191, filesize=30.4 K 2024-11-25T05:56:37,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/e53f3fd6372c42b79d85f4ce8eadb0d9 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/e53f3fd6372c42b79d85f4ce8eadb0d9 2024-11-25T05:56:37,318 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/e53f3fd6372c42b79d85f4ce8eadb0d9, entries=150, sequenceid=191, filesize=11.9 K 2024-11-25T05:56:37,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/2b031c403164448daeb3f61a048ca7ac as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/2b031c403164448daeb3f61a048ca7ac 2024-11-25T05:56:37,322 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/2b031c403164448daeb3f61a048ca7ac, entries=150, sequenceid=191, filesize=11.9 K 2024-11-25T05:56:37,323 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for ef2460dfb8241e76bc050e18474be77d in 1659ms, sequenceid=191, compaction requested=false 2024-11-25T05:56:37,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for ef2460dfb8241e76bc050e18474be77d: 2024-11-25T05:56:37,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:37,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-25T05:56:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-25T05:56:37,326 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-25T05:56:37,326 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.4100 sec 2024-11-25T05:56:37,327 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 4.4140 sec 2024-11-25T05:56:38,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:38,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ef2460dfb8241e76bc050e18474be77d 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-25T05:56:38,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=A 2024-11-25T05:56:38,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:56:38,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=B 2024-11-25T05:56:38,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:56:38,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=C 2024-11-25T05:56:38,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:56:38,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112558eff27fae5b40b9a7b1090ab22bf4bd_ef2460dfb8241e76bc050e18474be77d is 50, key is test_row_0/A:col10/1732514198176/Put/seqid=0 2024-11-25T05:56:38,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742230_1406 (size=14794) 2024-11-25T05:56:38,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:38,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514258206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:38,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:38,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:38,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:38,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:38,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514258209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:38,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:38,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:38,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:38,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:38,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514258322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:38,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:38,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:38,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:38,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:38,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514258322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:38,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:38,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514258532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:38,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:38,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:38,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:38,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:38,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44986 deadline: 1732514258541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:38,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:38,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514258542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:38,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:38,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:38,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:38,551 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_2' in ef2460dfb8241e76bc050e18474be77d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:56:38,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:38,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:38,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:38,588 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:56:38,591 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112558eff27fae5b40b9a7b1090ab22bf4bd_ef2460dfb8241e76bc050e18474be77d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112558eff27fae5b40b9a7b1090ab22bf4bd_ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:38,592 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/8d50265ad8c346fc9224e8a64a20a84a, store: [table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:38,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/8d50265ad8c346fc9224e8a64a20a84a is 175, key is test_row_0/A:col10/1732514198176/Put/seqid=0 2024-11-25T05:56:38,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742231_1407 (size=39749) 2024-11-25T05:56:38,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:38,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514258852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:38,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:38,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:38,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:38,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:38,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514258862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:38,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:38,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:38,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:38,996 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=231, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/8d50265ad8c346fc9224e8a64a20a84a 2024-11-25T05:56:39,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/9973f95f03184439919ce1b3c09e2d81 is 50, key is test_row_0/B:col10/1732514198176/Put/seqid=0 2024-11-25T05:56:39,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742232_1408 (size=12151) 2024-11-25T05:56:39,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:39,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514259372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:39,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:39,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:39,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:39,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:39,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514259382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:39,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:39,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:39,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:39,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/9973f95f03184439919ce1b3c09e2d81 2024-11-25T05:56:39,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/f0bbab63ea0846c3b499779adb251b59 is 50, key is test_row_0/C:col10/1732514198176/Put/seqid=0 2024-11-25T05:56:39,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742233_1409 (size=12151) 2024-11-25T05:56:39,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/f0bbab63ea0846c3b499779adb251b59 2024-11-25T05:56:39,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/8d50265ad8c346fc9224e8a64a20a84a as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/8d50265ad8c346fc9224e8a64a20a84a 2024-11-25T05:56:39,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/8d50265ad8c346fc9224e8a64a20a84a, entries=200, sequenceid=231, filesize=38.8 K 2024-11-25T05:56:39,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/9973f95f03184439919ce1b3c09e2d81 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/9973f95f03184439919ce1b3c09e2d81 2024-11-25T05:56:39,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/9973f95f03184439919ce1b3c09e2d81, entries=150, sequenceid=231, filesize=11.9 K 2024-11-25T05:56:39,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/f0bbab63ea0846c3b499779adb251b59 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/f0bbab63ea0846c3b499779adb251b59 2024-11-25T05:56:39,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/f0bbab63ea0846c3b499779adb251b59, entries=150, sequenceid=231, filesize=11.9 K 2024-11-25T05:56:39,830 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for ef2460dfb8241e76bc050e18474be77d in 1652ms, sequenceid=231, compaction requested=true 2024-11-25T05:56:39,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ef2460dfb8241e76bc050e18474be77d: 2024-11-25T05:56:39,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef2460dfb8241e76bc050e18474be77d:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:56:39,830 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:56:39,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:39,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef2460dfb8241e76bc050e18474be77d:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:56:39,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:39,830 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:56:39,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef2460dfb8241e76bc050e18474be77d:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:56:39,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:56:39,830 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:56:39,830 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:56:39,831 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): ef2460dfb8241e76bc050e18474be77d/B is initiating minor compaction (all files) 2024-11-25T05:56:39,831 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): ef2460dfb8241e76bc050e18474be77d/A is initiating minor compaction (all files) 2024-11-25T05:56:39,831 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef2460dfb8241e76bc050e18474be77d/A in TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:39,831 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef2460dfb8241e76bc050e18474be77d/B in TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:39,831 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/5ab476f341f54c188b704aa937b397c4, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/e53f3fd6372c42b79d85f4ce8eadb0d9, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/9973f95f03184439919ce1b3c09e2d81] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp, totalSize=35.9 K 2024-11-25T05:56:39,831 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/2db260a0c5c7406db7d38ce725e0bc2b, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/e69ff81d5c1a4757b258a6e15cee29f4, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/8d50265ad8c346fc9224e8a64a20a84a] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp, totalSize=99.9 K 2024-11-25T05:56:39,831 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:39,831 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. files: [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/2db260a0c5c7406db7d38ce725e0bc2b, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/e69ff81d5c1a4757b258a6e15cee29f4, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/8d50265ad8c346fc9224e8a64a20a84a] 2024-11-25T05:56:39,831 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ab476f341f54c188b704aa937b397c4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732514193577 2024-11-25T05:56:39,832 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2db260a0c5c7406db7d38ce725e0bc2b, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732514193577 2024-11-25T05:56:39,832 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting e53f3fd6372c42b79d85f4ce8eadb0d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732514194299 2024-11-25T05:56:39,833 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting e69ff81d5c1a4757b258a6e15cee29f4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732514194299 2024-11-25T05:56:39,833 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 9973f95f03184439919ce1b3c09e2d81, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732514195970 2024-11-25T05:56:39,833 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d50265ad8c346fc9224e8a64a20a84a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732514195970 2024-11-25T05:56:39,837 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:39,838 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef2460dfb8241e76bc050e18474be77d#B#compaction#350 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:56:39,839 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/6d7a679a3b6f4de4b2f80d47a1584f47 is 50, key is test_row_0/B:col10/1732514198176/Put/seqid=0 2024-11-25T05:56:39,842 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125e53dba622d2644cfbff591c9fbf96d7d_ef2460dfb8241e76bc050e18474be77d store=[table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:39,844 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125e53dba622d2644cfbff591c9fbf96d7d_ef2460dfb8241e76bc050e18474be77d, store=[table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:39,844 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125e53dba622d2644cfbff591c9fbf96d7d_ef2460dfb8241e76bc050e18474be77d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:39,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742234_1410 (size=12595) 2024-11-25T05:56:39,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742235_1411 (size=4469) 2024-11-25T05:56:40,250 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/6d7a679a3b6f4de4b2f80d47a1584f47 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/6d7a679a3b6f4de4b2f80d47a1584f47 2024-11-25T05:56:40,253 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef2460dfb8241e76bc050e18474be77d#A#compaction#351 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:56:40,254 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/fdce427124df4df3a1f8ca8b384f67f6 is 175, key is test_row_0/A:col10/1732514198176/Put/seqid=0 2024-11-25T05:56:40,254 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef2460dfb8241e76bc050e18474be77d/B of ef2460dfb8241e76bc050e18474be77d into 6d7a679a3b6f4de4b2f80d47a1584f47(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:56:40,254 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef2460dfb8241e76bc050e18474be77d: 2024-11-25T05:56:40,254 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., storeName=ef2460dfb8241e76bc050e18474be77d/B, priority=13, startTime=1732514199830; duration=0sec 2024-11-25T05:56:40,254 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:56:40,254 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef2460dfb8241e76bc050e18474be77d:B 2024-11-25T05:56:40,254 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:56:40,255 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:56:40,256 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): ef2460dfb8241e76bc050e18474be77d/C is initiating minor compaction (all files) 2024-11-25T05:56:40,256 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef2460dfb8241e76bc050e18474be77d/C in TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:40,256 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/c89b61141f1b4e41b5d01a198b516938, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/2b031c403164448daeb3f61a048ca7ac, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/f0bbab63ea0846c3b499779adb251b59] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp, totalSize=35.9 K 2024-11-25T05:56:40,256 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting c89b61141f1b4e41b5d01a198b516938, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732514193577 2024-11-25T05:56:40,256 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b031c403164448daeb3f61a048ca7ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732514194299 2024-11-25T05:56:40,257 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting f0bbab63ea0846c3b499779adb251b59, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732514195970 2024-11-25T05:56:40,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742236_1412 (size=31549) 2024-11-25T05:56:40,262 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef2460dfb8241e76bc050e18474be77d#C#compaction#352 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:56:40,262 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/75005d15561a4356aee6ac86e885ebe6 is 50, key is test_row_0/C:col10/1732514198176/Put/seqid=0 2024-11-25T05:56:40,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742237_1413 (size=12595) 2024-11-25T05:56:40,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:40,397 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ef2460dfb8241e76bc050e18474be77d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-25T05:56:40,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=A 2024-11-25T05:56:40,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:40,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=B 2024-11-25T05:56:40,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:40,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=C 2024-11-25T05:56:40,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:40,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112564f0f5a04f2647ea83c521f2a36b3826_ef2460dfb8241e76bc050e18474be77d is 50, key is test_row_0/A:col10/1732514200396/Put/seqid=0 2024-11-25T05:56:40,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742238_1414 (size=14794) 2024-11-25T05:56:40,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732514200443, max timestamp=1732514200473Num uniques 859; 2024-11-25T05:56:40,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732514200443, max timestamp=1732514200473Num uniques 859; 2024-11-25T05:56:40,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732514200443, max timestamp=1732514200473Num uniques 859; 2024-11-25T05:56:40,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-25T05:56:40,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-25T05:56:40,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-25T05:56:40,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:40,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514260544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:40,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:40,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:40,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:40,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:40,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514260550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:40,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:40,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:40,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:40,664 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/fdce427124df4df3a1f8ca8b384f67f6 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/fdce427124df4df3a1f8ca8b384f67f6 2024-11-25T05:56:40,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:40,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514260661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:40,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:40,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:40,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:40,667 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef2460dfb8241e76bc050e18474be77d/A of ef2460dfb8241e76bc050e18474be77d into fdce427124df4df3a1f8ca8b384f67f6(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:56:40,667 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef2460dfb8241e76bc050e18474be77d: 2024-11-25T05:56:40,667 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., storeName=ef2460dfb8241e76bc050e18474be77d/A, priority=13, startTime=1732514199830; duration=0sec 2024-11-25T05:56:40,667 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:40,667 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef2460dfb8241e76bc050e18474be77d:A 2024-11-25T05:56:40,675 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/75005d15561a4356aee6ac86e885ebe6 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/75005d15561a4356aee6ac86e885ebe6 2024-11-25T05:56:40,679 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef2460dfb8241e76bc050e18474be77d/C of ef2460dfb8241e76bc050e18474be77d into 75005d15561a4356aee6ac86e885ebe6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:56:40,679 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef2460dfb8241e76bc050e18474be77d: 2024-11-25T05:56:40,679 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., storeName=ef2460dfb8241e76bc050e18474be77d/C, priority=13, startTime=1732514199830; duration=0sec 2024-11-25T05:56:40,679 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:40,679 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef2460dfb8241e76bc050e18474be77d:C 2024-11-25T05:56:40,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:40,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514260672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:40,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:40,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:40,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:40,818 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:56:40,822 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112564f0f5a04f2647ea83c521f2a36b3826_ef2460dfb8241e76bc050e18474be77d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112564f0f5a04f2647ea83c521f2a36b3826_ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:40,823 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/584c9d28057e44f8ba99abd6ffbfe661, store: [table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:40,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/584c9d28057e44f8ba99abd6ffbfe661 is 175, key is test_row_0/A:col10/1732514200396/Put/seqid=0 2024-11-25T05:56:40,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742239_1415 (size=39749) 2024-11-25T05:56:40,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:40,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514260872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:40,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:40,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:40,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:40,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:40,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514260891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:40,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:40,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:40,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:41,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-25T05:56:41,123 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:56:41,123 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:56:41,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-25T05:56:41,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-25T05:56:41,125 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:56:41,125 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:56:41,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:56:41,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:41,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514261192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:41,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:41,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:41,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:41,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:41,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514261212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:41,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:41,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:41,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:41,229 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=242, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/584c9d28057e44f8ba99abd6ffbfe661 2024-11-25T05:56:41,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-25T05:56:41,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/a7ffbfc31b724db3892db08cdf2f635a is 50, key is test_row_0/B:col10/1732514200396/Put/seqid=0 2024-11-25T05:56:41,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742240_1416 (size=12151) 2024-11-25T05:56:41,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/a7ffbfc31b724db3892db08cdf2f635a 2024-11-25T05:56:41,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/3b417e60f7d94035a548c79eb442dc81 is 50, key is test_row_0/C:col10/1732514200396/Put/seqid=0 2024-11-25T05:56:41,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742241_1417 (size=12151) 2024-11-25T05:56:41,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T05:56:41,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:41,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:41,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:41,278 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:41,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:41,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:41,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T05:56:41,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:41,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:41,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:41,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:41,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:41,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:41,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-25T05:56:41,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T05:56:41,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:41,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:41,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:41,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:41,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:41,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:41,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/3b417e60f7d94035a548c79eb442dc81 2024-11-25T05:56:41,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/584c9d28057e44f8ba99abd6ffbfe661 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/584c9d28057e44f8ba99abd6ffbfe661 2024-11-25T05:56:41,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/584c9d28057e44f8ba99abd6ffbfe661, entries=200, sequenceid=242, filesize=38.8 K 2024-11-25T05:56:41,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/a7ffbfc31b724db3892db08cdf2f635a as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/a7ffbfc31b724db3892db08cdf2f635a 2024-11-25T05:56:41,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/a7ffbfc31b724db3892db08cdf2f635a, entries=150, sequenceid=242, filesize=11.9 K 2024-11-25T05:56:41,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/3b417e60f7d94035a548c79eb442dc81 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/3b417e60f7d94035a548c79eb442dc81 2024-11-25T05:56:41,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/3b417e60f7d94035a548c79eb442dc81, entries=150, sequenceid=242, filesize=11.9 K 2024-11-25T05:56:41,675 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for ef2460dfb8241e76bc050e18474be77d in 1278ms, sequenceid=242, compaction requested=false 2024-11-25T05:56:41,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ef2460dfb8241e76bc050e18474be77d: 2024-11-25T05:56:41,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:41,717 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ef2460dfb8241e76bc050e18474be77d 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-25T05:56:41,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=A 2024-11-25T05:56:41,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:56:41,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=B 2024-11-25T05:56:41,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:56:41,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=C 2024-11-25T05:56:41,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:56:41,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c2ff5642bbed41b580272969c76122b2_ef2460dfb8241e76bc050e18474be77d is 50, key is test_row_0/A:col10/1732514200506/Put/seqid=0 2024-11-25T05:56:41,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742242_1418 (size=14994) 2024-11-25T05:56:41,729 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:56:41,732 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c2ff5642bbed41b580272969c76122b2_ef2460dfb8241e76bc050e18474be77d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c2ff5642bbed41b580272969c76122b2_ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:41,732 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/72c5191aacd84a53a89803d5cd259fba, store: [table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:41,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/72c5191aacd84a53a89803d5cd259fba is 175, key is test_row_0/A:col10/1732514200506/Put/seqid=0 2024-11-25T05:56:41,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T05:56:41,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:41,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:41,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:41,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:41,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:41,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:41,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742243_1419 (size=39949) 2024-11-25T05:56:41,741 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=280, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/72c5191aacd84a53a89803d5cd259fba 2024-11-25T05:56:41,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/48d88adad3b146828e1e743b929989e0 is 50, key is test_row_0/B:col10/1732514200506/Put/seqid=0 2024-11-25T05:56:41,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742244_1420 (size=12301) 2024-11-25T05:56:41,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-25T05:56:41,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:41,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514261754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:41,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:41,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:41,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:41,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:41,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514261755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:41,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:41,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:41,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:41,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:41,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514261862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:41,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:41,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:41,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:41,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T05:56:41,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:41,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:41,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:41,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:41,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:41,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:41,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:41,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514261882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:41,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:41,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:41,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:42,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T05:56:42,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:42,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:42,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:42,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:42,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:42,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:42,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:42,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514262072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:42,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:42,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:42,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:42,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:42,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514262102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:42,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:42,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:42,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:42,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/48d88adad3b146828e1e743b929989e0 2024-11-25T05:56:42,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/c3b1c81579724ed2a3044351bc6ee339 is 50, key is test_row_0/C:col10/1732514200506/Put/seqid=0 2024-11-25T05:56:42,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742245_1421 (size=12301) 2024-11-25T05:56:42,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T05:56:42,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:42,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:42,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:42,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:42,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:42,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:42,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-25T05:56:42,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T05:56:42,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:42,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:42,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:42,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:42,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:42,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:42,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:42,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514262392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:42,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:42,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:42,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:42,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:42,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514262412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:42,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:42,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:42,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:42,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T05:56:42,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:42,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:42,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:42,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:42,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:42,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:56:42,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/c3b1c81579724ed2a3044351bc6ee339 2024-11-25T05:56:42,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:42,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/72c5191aacd84a53a89803d5cd259fba as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/72c5191aacd84a53a89803d5cd259fba 2024-11-25T05:56:42,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44986 deadline: 1732514262562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:42,568 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_2' in ef2460dfb8241e76bc050e18474be77d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8250 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:56:42,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:42,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:42,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:42,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/72c5191aacd84a53a89803d5cd259fba, entries=200, sequenceid=280, filesize=39.0 K 2024-11-25T05:56:42,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/48d88adad3b146828e1e743b929989e0 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/48d88adad3b146828e1e743b929989e0 2024-11-25T05:56:42,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/48d88adad3b146828e1e743b929989e0, entries=150, sequenceid=280, filesize=12.0 K 2024-11-25T05:56:42,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/c3b1c81579724ed2a3044351bc6ee339 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/c3b1c81579724ed2a3044351bc6ee339 2024-11-25T05:56:42,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/c3b1c81579724ed2a3044351bc6ee339, entries=150, sequenceid=280, filesize=12.0 K 2024-11-25T05:56:42,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=40.25 KB/41220 for ef2460dfb8241e76bc050e18474be77d in 862ms, sequenceid=280, compaction requested=true 2024-11-25T05:56:42,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ef2460dfb8241e76bc050e18474be77d: 2024-11-25T05:56:42,579 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:56:42,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef2460dfb8241e76bc050e18474be77d:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:56:42,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:42,579 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:56:42,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef2460dfb8241e76bc050e18474be77d:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:56:42,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:42,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef2460dfb8241e76bc050e18474be77d:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:56:42,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:56:42,580 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111247 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:56:42,580 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): ef2460dfb8241e76bc050e18474be77d/A is initiating minor compaction (all files) 2024-11-25T05:56:42,580 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef2460dfb8241e76bc050e18474be77d/A in TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:42,580 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/fdce427124df4df3a1f8ca8b384f67f6, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/584c9d28057e44f8ba99abd6ffbfe661, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/72c5191aacd84a53a89803d5cd259fba] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp, totalSize=108.6 K 2024-11-25T05:56:42,580 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:42,580 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. files: [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/fdce427124df4df3a1f8ca8b384f67f6, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/584c9d28057e44f8ba99abd6ffbfe661, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/72c5191aacd84a53a89803d5cd259fba] 2024-11-25T05:56:42,581 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdce427124df4df3a1f8ca8b384f67f6, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732514195970 2024-11-25T05:56:42,581 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:56:42,581 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): ef2460dfb8241e76bc050e18474be77d/B is initiating minor compaction (all files) 2024-11-25T05:56:42,581 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef2460dfb8241e76bc050e18474be77d/B in TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:42,581 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/6d7a679a3b6f4de4b2f80d47a1584f47, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/a7ffbfc31b724db3892db08cdf2f635a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/48d88adad3b146828e1e743b929989e0] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp, totalSize=36.2 K 2024-11-25T05:56:42,581 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 584c9d28057e44f8ba99abd6ffbfe661, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732514198198 2024-11-25T05:56:42,581 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d7a679a3b6f4de4b2f80d47a1584f47, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732514195970 2024-11-25T05:56:42,582 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72c5191aacd84a53a89803d5cd259fba, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732514200506 2024-11-25T05:56:42,582 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting a7ffbfc31b724db3892db08cdf2f635a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732514198198 2024-11-25T05:56:42,582 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 48d88adad3b146828e1e743b929989e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732514200506 2024-11-25T05:56:42,586 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:42,588 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125c8a4aa63e89749b8b9c845f60b4b1b42_ef2460dfb8241e76bc050e18474be77d store=[table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:42,588 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef2460dfb8241e76bc050e18474be77d#B#compaction#359 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:56:42,589 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/341f0ec12071448dbab214f119c0f81e is 50, key is test_row_0/B:col10/1732514200506/Put/seqid=0 2024-11-25T05:56:42,591 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125c8a4aa63e89749b8b9c845f60b4b1b42_ef2460dfb8241e76bc050e18474be77d, store=[table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:42,591 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c8a4aa63e89749b8b9c845f60b4b1b42_ef2460dfb8241e76bc050e18474be77d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:42,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742246_1422 (size=4469) 2024-11-25T05:56:42,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742247_1423 (size=12847) 2024-11-25T05:56:42,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T05:56:42,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:42,651 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing ef2460dfb8241e76bc050e18474be77d 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-25T05:56:42,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=A 2024-11-25T05:56:42,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:42,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=B 2024-11-25T05:56:42,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:42,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=C 2024-11-25T05:56:42,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:42,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125a3f1a4d4f6e44151b419d2e9289b9809_ef2460dfb8241e76bc050e18474be77d is 50, key is test_row_0/A:col10/1732514201752/Put/seqid=0 2024-11-25T05:56:42,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742248_1424 (size=12454) 2024-11-25T05:56:42,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:42,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. as already flushing 2024-11-25T05:56:42,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732514202963, max timestamp=1732514202988Num uniques 859; 2024-11-25T05:56:42,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T05:56:42,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732514202963, max timestamp=1732514202988Num uniques 859; 2024-11-25T05:56:42,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732514202963, max timestamp=1732514202988Num uniques 859; 2024-11-25T05:56:42,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T05:56:42,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T05:56:43,005 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef2460dfb8241e76bc050e18474be77d#A#compaction#360 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:56:43,005 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/89f01458742f4ad297ac71e9849c6179 is 175, key is test_row_0/A:col10/1732514200506/Put/seqid=0 2024-11-25T05:56:43,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742249_1425 (size=31801) 2024-11-25T05:56:43,008 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/341f0ec12071448dbab214f119c0f81e as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/341f0ec12071448dbab214f119c0f81e 2024-11-25T05:56:43,012 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef2460dfb8241e76bc050e18474be77d/B of ef2460dfb8241e76bc050e18474be77d into 341f0ec12071448dbab214f119c0f81e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:56:43,012 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef2460dfb8241e76bc050e18474be77d: 2024-11-25T05:56:43,012 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., storeName=ef2460dfb8241e76bc050e18474be77d/B, priority=13, startTime=1732514202579; duration=0sec 2024-11-25T05:56:43,012 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:56:43,012 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef2460dfb8241e76bc050e18474be77d:B 2024-11-25T05:56:43,012 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:56:43,013 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:56:43,013 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): ef2460dfb8241e76bc050e18474be77d/C is initiating minor compaction (all files) 2024-11-25T05:56:43,013 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef2460dfb8241e76bc050e18474be77d/C in TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:43,013 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/75005d15561a4356aee6ac86e885ebe6, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/3b417e60f7d94035a548c79eb442dc81, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/c3b1c81579724ed2a3044351bc6ee339] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp, totalSize=36.2 K 2024-11-25T05:56:43,013 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 75005d15561a4356aee6ac86e885ebe6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732514195970 2024-11-25T05:56:43,014 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b417e60f7d94035a548c79eb442dc81, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732514198198 2024-11-25T05:56:43,014 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting c3b1c81579724ed2a3044351bc6ee339, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732514200506 2024-11-25T05:56:43,020 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef2460dfb8241e76bc050e18474be77d#C#compaction#362 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:56:43,020 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/a2f6b0b961894f1d96a91f2eeb60d58f is 50, key is test_row_0/C:col10/1732514200506/Put/seqid=0 2024-11-25T05:56:43,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742250_1426 (size=12847) 2024-11-25T05:56:43,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:56:43,066 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125a3f1a4d4f6e44151b419d2e9289b9809_ef2460dfb8241e76bc050e18474be77d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125a3f1a4d4f6e44151b419d2e9289b9809_ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:43,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/be102a3893ff452fabce5276fc0a1859, store: [table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:43,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/be102a3893ff452fabce5276fc0a1859 is 175, key is test_row_0/A:col10/1732514201752/Put/seqid=0 2024-11-25T05:56:43,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742251_1427 (size=31255) 2024-11-25T05:56:43,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:43,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514263081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:43,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:43,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:43,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:43,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:43,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514263083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:43,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:43,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:43,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:43,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:43,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514263192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:43,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:43,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:43,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:43,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:43,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514263202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:43,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:43,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:43,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:43,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-25T05:56:43,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:43,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514263402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:43,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:43,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:43,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:43,412 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/89f01458742f4ad297ac71e9849c6179 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/89f01458742f4ad297ac71e9849c6179 2024-11-25T05:56:43,416 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef2460dfb8241e76bc050e18474be77d/A of ef2460dfb8241e76bc050e18474be77d into 89f01458742f4ad297ac71e9849c6179(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:56:43,416 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef2460dfb8241e76bc050e18474be77d: 2024-11-25T05:56:43,416 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., storeName=ef2460dfb8241e76bc050e18474be77d/A, priority=13, startTime=1732514202579; duration=0sec 2024-11-25T05:56:43,416 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:43,416 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef2460dfb8241e76bc050e18474be77d:A 2024-11-25T05:56:43,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:43,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514263412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:43,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:43,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:43,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:43,426 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/a2f6b0b961894f1d96a91f2eeb60d58f as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/a2f6b0b961894f1d96a91f2eeb60d58f 2024-11-25T05:56:43,430 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef2460dfb8241e76bc050e18474be77d/C of ef2460dfb8241e76bc050e18474be77d into a2f6b0b961894f1d96a91f2eeb60d58f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:56:43,430 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef2460dfb8241e76bc050e18474be77d: 2024-11-25T05:56:43,430 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., storeName=ef2460dfb8241e76bc050e18474be77d/C, priority=13, startTime=1732514202579; duration=0sec 2024-11-25T05:56:43,430 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:43,430 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef2460dfb8241e76bc050e18474be77d:C 2024-11-25T05:56:43,471 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=289, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/be102a3893ff452fabce5276fc0a1859 2024-11-25T05:56:43,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/91dcc0cf8ea444d0b67043a306813ad7 is 50, key is test_row_0/B:col10/1732514201752/Put/seqid=0 2024-11-25T05:56:43,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742252_1428 (size=12301) 2024-11-25T05:56:43,486 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/91dcc0cf8ea444d0b67043a306813ad7 2024-11-25T05:56:43,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/d75b91e661fd4cbfa0284941ab9955af is 50, key is test_row_0/C:col10/1732514201752/Put/seqid=0 2024-11-25T05:56:43,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742253_1429 (size=12301) 2024-11-25T05:56:43,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514263722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:43,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514263731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:43,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:43,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:43,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:43,897 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/d75b91e661fd4cbfa0284941ab9955af 2024-11-25T05:56:43,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/be102a3893ff452fabce5276fc0a1859 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/be102a3893ff452fabce5276fc0a1859 2024-11-25T05:56:43,904 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/be102a3893ff452fabce5276fc0a1859, entries=150, sequenceid=289, filesize=30.5 K 2024-11-25T05:56:43,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/91dcc0cf8ea444d0b67043a306813ad7 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/91dcc0cf8ea444d0b67043a306813ad7 2024-11-25T05:56:43,907 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/91dcc0cf8ea444d0b67043a306813ad7, entries=150, sequenceid=289, filesize=12.0 K 2024-11-25T05:56:43,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/d75b91e661fd4cbfa0284941ab9955af as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/d75b91e661fd4cbfa0284941ab9955af 2024-11-25T05:56:43,912 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/d75b91e661fd4cbfa0284941ab9955af, entries=150, sequenceid=289, filesize=12.0 K 2024-11-25T05:56:43,913 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for ef2460dfb8241e76bc050e18474be77d in 1261ms, sequenceid=289, compaction requested=false 2024-11-25T05:56:43,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for ef2460dfb8241e76bc050e18474be77d: 2024-11-25T05:56:43,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:43,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-25T05:56:43,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-25T05:56:43,916 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-25T05:56:43,916 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7890 sec 2024-11-25T05:56:43,917 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.7930 sec 2024-11-25T05:56:44,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:44,248 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ef2460dfb8241e76bc050e18474be77d 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-25T05:56:44,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=A 2024-11-25T05:56:44,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-25T05:56:44,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=B 2024-11-25T05:56:44,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-25T05:56:44,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=C 2024-11-25T05:56:44,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-25T05:56:44,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125a1b396fafc23460aaa2bac920504a510_ef2460dfb8241e76bc050e18474be77d is 50, key is test_row_0/A:col10/1732514203079/Put/seqid=0 2024-11-25T05:56:44,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742254_1430 (size=14994) 2024-11-25T05:56:44,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:44,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514264272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:44,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:44,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:44,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:44,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:44,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514264273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:44,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:44,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:44,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:44,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:44,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514264381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:44,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:44,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:44,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:44,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514264392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:44,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:44,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:44,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:44,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:44,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514264592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:44,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:44,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:44,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:44,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:44,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514264601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:44,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:44,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:44,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:44,611 INFO [Thread-1631 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1631. 2024-11-25T05:56:44,611 DEBUG [Thread-1631 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:56:44,611 DEBUG [Thread-1631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:44,611 DEBUG [Thread-1631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:44,611 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:56:44,612 INFO [Thread-1627 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1627. 2024-11-25T05:56:44,612 DEBUG [Thread-1627 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:56:44,612 DEBUG [Thread-1627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:44,612 DEBUG [Thread-1627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:44,612 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:56:44,620 INFO [Thread-1625 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1625. 2024-11-25T05:56:44,620 DEBUG [Thread-1625 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:56:44,620 DEBUG [Thread-1625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:44,620 DEBUG [Thread-1625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:44,620 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:56:44,622 INFO [Thread-1629 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1629. 2024-11-25T05:56:44,622 DEBUG [Thread-1629 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:56:44,622 DEBUG [Thread-1629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:44,622 DEBUG [Thread-1629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:44,622 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:56:44,624 INFO [Thread-1623 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1623. 2024-11-25T05:56:44,624 DEBUG [Thread-1623 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:56:44,624 DEBUG [Thread-1623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:44,624 DEBUG [Thread-1623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:44,624 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:56:44,659 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:56:44,664 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125a1b396fafc23460aaa2bac920504a510_ef2460dfb8241e76bc050e18474be77d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125a1b396fafc23460aaa2bac920504a510_ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:44,665 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/fb00031b3a9d4ca997a64c858f3867cc, store: [table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:44,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/fb00031b3a9d4ca997a64c858f3867cc is 175, key is test_row_0/A:col10/1732514203079/Put/seqid=0 2024-11-25T05:56:44,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742255_1431 (size=39949) 2024-11-25T05:56:44,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:44,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514264913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:44,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:44,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:44,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:44,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:44,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514264923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:44,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:44,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:44,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:45,073 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=330, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/fb00031b3a9d4ca997a64c858f3867cc 2024-11-25T05:56:45,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/58cbff72752d482da8c669a956313736 is 50, key is test_row_0/B:col10/1732514203079/Put/seqid=0 2024-11-25T05:56:45,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742256_1432 (size=12301) 2024-11-25T05:56:45,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-25T05:56:45,294 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:56:45,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:45,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44934 deadline: 1732514265343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:45,345 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_1' in ef2460dfb8241e76bc050e18474be77d of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18283 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:56:45,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:45,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:45,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:45,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:45,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44954 deadline: 1732514265382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:45,383 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_1' in ef2460dfb8241e76bc050e18474be77d of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18322 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:56:45,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:45,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:45,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:45,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:45,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44984 deadline: 1732514265422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:45,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:45,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:45,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:45,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:45,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44952 deadline: 1732514265433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:45,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:45,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:45,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:56:45,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/58cbff72752d482da8c669a956313736 2024-11-25T05:56:45,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/e926848829124d78ac19358814a4ad0b is 50, key is test_row_0/C:col10/1732514203079/Put/seqid=0 2024-11-25T05:56:45,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742257_1433 (size=12301) 2024-11-25T05:56:45,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/e926848829124d78ac19358814a4ad0b 2024-11-25T05:56:45,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/fb00031b3a9d4ca997a64c858f3867cc as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/fb00031b3a9d4ca997a64c858f3867cc 2024-11-25T05:56:45,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/fb00031b3a9d4ca997a64c858f3867cc, entries=200, sequenceid=330, filesize=39.0 K 2024-11-25T05:56:45,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/58cbff72752d482da8c669a956313736 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/58cbff72752d482da8c669a956313736 2024-11-25T05:56:45,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/58cbff72752d482da8c669a956313736, entries=150, sequenceid=330, filesize=12.0 K 2024-11-25T05:56:45,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/e926848829124d78ac19358814a4ad0b as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/e926848829124d78ac19358814a4ad0b 2024-11-25T05:56:45,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/e926848829124d78ac19358814a4ad0b, entries=150, sequenceid=330, filesize=12.0 K 2024-11-25T05:56:45,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for ef2460dfb8241e76bc050e18474be77d in 1684ms, sequenceid=330, compaction requested=true 2024-11-25T05:56:45,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ef2460dfb8241e76bc050e18474be77d: 2024-11-25T05:56:45,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef2460dfb8241e76bc050e18474be77d:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:56:45,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:45,932 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:56:45,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef2460dfb8241e76bc050e18474be77d:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:56:45,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:45,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef2460dfb8241e76bc050e18474be77d:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:56:45,932 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:56:45,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:56:45,933 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:56:45,933 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): ef2460dfb8241e76bc050e18474be77d/A is initiating minor compaction (all files) 2024-11-25T05:56:45,933 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:56:45,933 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef2460dfb8241e76bc050e18474be77d/A in TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:45,933 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): ef2460dfb8241e76bc050e18474be77d/B is initiating minor compaction (all files) 2024-11-25T05:56:45,933 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/89f01458742f4ad297ac71e9849c6179, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/be102a3893ff452fabce5276fc0a1859, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/fb00031b3a9d4ca997a64c858f3867cc] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp, totalSize=100.6 K 2024-11-25T05:56:45,933 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef2460dfb8241e76bc050e18474be77d/B in TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:45,933 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:45,933 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. files: [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/89f01458742f4ad297ac71e9849c6179, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/be102a3893ff452fabce5276fc0a1859, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/fb00031b3a9d4ca997a64c858f3867cc] 2024-11-25T05:56:45,933 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/341f0ec12071448dbab214f119c0f81e, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/91dcc0cf8ea444d0b67043a306813ad7, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/58cbff72752d482da8c669a956313736] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp, totalSize=36.6 K 2024-11-25T05:56:45,933 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89f01458742f4ad297ac71e9849c6179, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732514200506 2024-11-25T05:56:45,933 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 341f0ec12071448dbab214f119c0f81e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732514200506 2024-11-25T05:56:45,933 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting be102a3893ff452fabce5276fc0a1859, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732514201737 2024-11-25T05:56:45,933 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 91dcc0cf8ea444d0b67043a306813ad7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732514201737 2024-11-25T05:56:45,934 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb00031b3a9d4ca997a64c858f3867cc, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732514203064 2024-11-25T05:56:45,934 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 58cbff72752d482da8c669a956313736, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732514203079 2024-11-25T05:56:45,938 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:45,939 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125fc1647ceadbe415685006a39e54c8ba7_ef2460dfb8241e76bc050e18474be77d store=[table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:45,939 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef2460dfb8241e76bc050e18474be77d#B#compaction#368 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:56:45,939 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/6cc034a1d36044d7aa7daf6ca43660e3 is 50, key is test_row_0/B:col10/1732514203079/Put/seqid=0 2024-11-25T05:56:45,942 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125fc1647ceadbe415685006a39e54c8ba7_ef2460dfb8241e76bc050e18474be77d, store=[table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:45,942 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125fc1647ceadbe415685006a39e54c8ba7_ef2460dfb8241e76bc050e18474be77d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:45,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742258_1434 (size=12949) 2024-11-25T05:56:45,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742259_1435 (size=4469) 2024-11-25T05:56:46,348 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef2460dfb8241e76bc050e18474be77d#A#compaction#369 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:56:46,349 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/30c02f3bd6c94256a2857eb18565ef90 is 175, key is test_row_0/A:col10/1732514203079/Put/seqid=0 2024-11-25T05:56:46,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742260_1436 (size=31903) 2024-11-25T05:56:46,356 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/6cc034a1d36044d7aa7daf6ca43660e3 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/6cc034a1d36044d7aa7daf6ca43660e3 2024-11-25T05:56:46,360 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef2460dfb8241e76bc050e18474be77d/B of ef2460dfb8241e76bc050e18474be77d into 6cc034a1d36044d7aa7daf6ca43660e3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:56:46,360 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef2460dfb8241e76bc050e18474be77d: 2024-11-25T05:56:46,360 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., storeName=ef2460dfb8241e76bc050e18474be77d/B, priority=13, startTime=1732514205932; duration=0sec 2024-11-25T05:56:46,360 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:56:46,360 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef2460dfb8241e76bc050e18474be77d:B 2024-11-25T05:56:46,360 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:56:46,361 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:56:46,361 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): ef2460dfb8241e76bc050e18474be77d/C is initiating minor compaction (all files) 2024-11-25T05:56:46,361 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef2460dfb8241e76bc050e18474be77d/C in TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:46,361 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/a2f6b0b961894f1d96a91f2eeb60d58f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/d75b91e661fd4cbfa0284941ab9955af, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/e926848829124d78ac19358814a4ad0b] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp, totalSize=36.6 K 2024-11-25T05:56:46,361 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting a2f6b0b961894f1d96a91f2eeb60d58f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732514200506 2024-11-25T05:56:46,362 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting d75b91e661fd4cbfa0284941ab9955af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732514201737 2024-11-25T05:56:46,362 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting e926848829124d78ac19358814a4ad0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732514203079 2024-11-25T05:56:46,369 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef2460dfb8241e76bc050e18474be77d#C#compaction#370 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:56:46,370 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/5995e1a8b8774c1ba5260a653b9e3a3b is 50, key is test_row_0/C:col10/1732514203079/Put/seqid=0 2024-11-25T05:56:46,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742261_1437 (size=12949) 2024-11-25T05:56:46,433 INFO [Thread-1612 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1612. 2024-11-25T05:56:46,434 DEBUG [Thread-1612 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:56:46,434 DEBUG [Thread-1612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:46,434 DEBUG [Thread-1612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:46,434 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:56:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:46,444 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ef2460dfb8241e76bc050e18474be77d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-25T05:56:46,444 INFO [Thread-1618 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1618. 2024-11-25T05:56:46,444 DEBUG [Thread-1618 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:56:46,444 DEBUG [Thread-1618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:46,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=A 2024-11-25T05:56:46,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:46,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=B 2024-11-25T05:56:46,444 DEBUG [Thread-1618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:46,444 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:56:46,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:46,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=C 2024-11-25T05:56:46,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:46,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125e9d6dfdcecc244ee83b7a64822f4a330_ef2460dfb8241e76bc050e18474be77d is 50, key is test_row_0/A:col10/1732514204270/Put/seqid=0 2024-11-25T05:56:46,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742262_1438 (size=12454) 2024-11-25T05:56:46,764 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/30c02f3bd6c94256a2857eb18565ef90 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/30c02f3bd6c94256a2857eb18565ef90 2024-11-25T05:56:46,771 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef2460dfb8241e76bc050e18474be77d/A of ef2460dfb8241e76bc050e18474be77d into 30c02f3bd6c94256a2857eb18565ef90(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:56:46,771 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef2460dfb8241e76bc050e18474be77d: 2024-11-25T05:56:46,771 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., storeName=ef2460dfb8241e76bc050e18474be77d/A, priority=13, startTime=1732514205932; duration=0sec 2024-11-25T05:56:46,772 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:46,772 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef2460dfb8241e76bc050e18474be77d:A 2024-11-25T05:56:46,780 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/5995e1a8b8774c1ba5260a653b9e3a3b as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/5995e1a8b8774c1ba5260a653b9e3a3b 2024-11-25T05:56:46,787 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef2460dfb8241e76bc050e18474be77d/C of ef2460dfb8241e76bc050e18474be77d into 5995e1a8b8774c1ba5260a653b9e3a3b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:56:46,787 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef2460dfb8241e76bc050e18474be77d: 2024-11-25T05:56:46,787 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., storeName=ef2460dfb8241e76bc050e18474be77d/C, priority=13, startTime=1732514205932; duration=0sec 2024-11-25T05:56:46,788 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:56:46,788 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef2460dfb8241e76bc050e18474be77d:C 2024-11-25T05:56:46,854 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:56:46,862 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125e9d6dfdcecc244ee83b7a64822f4a330_ef2460dfb8241e76bc050e18474be77d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125e9d6dfdcecc244ee83b7a64822f4a330_ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:46,863 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/296e0a112ac5451990b56a7092c32af3, store: [table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:46,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/296e0a112ac5451990b56a7092c32af3 is 175, key is test_row_0/A:col10/1732514204270/Put/seqid=0 2024-11-25T05:56:46,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742263_1439 (size=31255) 2024-11-25T05:56:47,267 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=341, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/296e0a112ac5451990b56a7092c32af3 2024-11-25T05:56:47,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/0a2f81d1c8344de9850fc4011114855e is 50, key is test_row_0/B:col10/1732514204270/Put/seqid=0 2024-11-25T05:56:47,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742264_1440 (size=12301) 2024-11-25T05:56:47,545 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T05:56:47,686 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/0a2f81d1c8344de9850fc4011114855e 2024-11-25T05:56:47,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/43760394cbc04f6bb9ac2ae805f4cd4e is 50, key is test_row_0/C:col10/1732514204270/Put/seqid=0 2024-11-25T05:56:47,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742265_1441 (size=12301) 2024-11-25T05:56:48,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/43760394cbc04f6bb9ac2ae805f4cd4e 2024-11-25T05:56:48,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/296e0a112ac5451990b56a7092c32af3 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/296e0a112ac5451990b56a7092c32af3 2024-11-25T05:56:48,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/296e0a112ac5451990b56a7092c32af3, entries=150, sequenceid=341, filesize=30.5 K 2024-11-25T05:56:48,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/0a2f81d1c8344de9850fc4011114855e as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/0a2f81d1c8344de9850fc4011114855e 2024-11-25T05:56:48,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/0a2f81d1c8344de9850fc4011114855e, entries=150, sequenceid=341, filesize=12.0 K 2024-11-25T05:56:48,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/43760394cbc04f6bb9ac2ae805f4cd4e as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/43760394cbc04f6bb9ac2ae805f4cd4e 2024-11-25T05:56:48,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/43760394cbc04f6bb9ac2ae805f4cd4e, entries=150, sequenceid=341, filesize=12.0 K 2024-11-25T05:56:48,126 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for ef2460dfb8241e76bc050e18474be77d in 1683ms, sequenceid=341, compaction requested=false 2024-11-25T05:56:48,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ef2460dfb8241e76bc050e18474be77d: 2024-11-25T05:56:51,672 WARN [regionserver/8ef925b832e3:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 2, running: 1 2024-11-25T05:56:51,675 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/04f0663834584962b1b5c03efefb68a6, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/c629c432b45245b1a8b6b4bdbdea793f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/fcaaad4739ca4a0094a63b41e0c44b7e, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/f9290140b32840fd9ea92287fdde5770, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/979e8d987041446c96c1222d08e759ae, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/85e0727b5ff24c3ea8b7d1e1c3f3c671, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/4d4b0bbf03bc46b59386153f52f35ac7, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/969e3b9557d94a2291edf3e96bd189ca, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/04ee38a92d83411ea8c62b9617672591, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/a0aad8fc67e1434cbe53c44cd1785221, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/2db260a0c5c7406db7d38ce725e0bc2b, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/e69ff81d5c1a4757b258a6e15cee29f4, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/8d50265ad8c346fc9224e8a64a20a84a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/fdce427124df4df3a1f8ca8b384f67f6, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/584c9d28057e44f8ba99abd6ffbfe661, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/72c5191aacd84a53a89803d5cd259fba, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/89f01458742f4ad297ac71e9849c6179, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/be102a3893ff452fabce5276fc0a1859, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/fb00031b3a9d4ca997a64c858f3867cc] to archive 2024-11-25T05:56:51,677 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T05:56:51,679 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/04f0663834584962b1b5c03efefb68a6 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/04f0663834584962b1b5c03efefb68a6 2024-11-25T05:56:51,681 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/c629c432b45245b1a8b6b4bdbdea793f to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/c629c432b45245b1a8b6b4bdbdea793f 2024-11-25T05:56:51,682 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/fcaaad4739ca4a0094a63b41e0c44b7e to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/fcaaad4739ca4a0094a63b41e0c44b7e 2024-11-25T05:56:51,684 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/f9290140b32840fd9ea92287fdde5770 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/f9290140b32840fd9ea92287fdde5770 2024-11-25T05:56:51,685 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/979e8d987041446c96c1222d08e759ae to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/979e8d987041446c96c1222d08e759ae 2024-11-25T05:56:51,686 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/85e0727b5ff24c3ea8b7d1e1c3f3c671 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/85e0727b5ff24c3ea8b7d1e1c3f3c671 2024-11-25T05:56:51,688 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/4d4b0bbf03bc46b59386153f52f35ac7 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/4d4b0bbf03bc46b59386153f52f35ac7 2024-11-25T05:56:51,689 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/969e3b9557d94a2291edf3e96bd189ca to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/969e3b9557d94a2291edf3e96bd189ca 2024-11-25T05:56:51,691 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/04ee38a92d83411ea8c62b9617672591 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/04ee38a92d83411ea8c62b9617672591 2024-11-25T05:56:51,692 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/a0aad8fc67e1434cbe53c44cd1785221 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/a0aad8fc67e1434cbe53c44cd1785221 2024-11-25T05:56:51,694 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/2db260a0c5c7406db7d38ce725e0bc2b to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/2db260a0c5c7406db7d38ce725e0bc2b 2024-11-25T05:56:51,695 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/e69ff81d5c1a4757b258a6e15cee29f4 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/e69ff81d5c1a4757b258a6e15cee29f4 2024-11-25T05:56:51,696 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/8d50265ad8c346fc9224e8a64a20a84a to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/8d50265ad8c346fc9224e8a64a20a84a 2024-11-25T05:56:51,698 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/fdce427124df4df3a1f8ca8b384f67f6 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/fdce427124df4df3a1f8ca8b384f67f6 2024-11-25T05:56:51,699 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/584c9d28057e44f8ba99abd6ffbfe661 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/584c9d28057e44f8ba99abd6ffbfe661 2024-11-25T05:56:51,700 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/72c5191aacd84a53a89803d5cd259fba to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/72c5191aacd84a53a89803d5cd259fba 2024-11-25T05:56:51,702 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/89f01458742f4ad297ac71e9849c6179 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/89f01458742f4ad297ac71e9849c6179 2024-11-25T05:56:51,703 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/be102a3893ff452fabce5276fc0a1859 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/be102a3893ff452fabce5276fc0a1859 2024-11-25T05:56:51,704 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/fb00031b3a9d4ca997a64c858f3867cc to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/fb00031b3a9d4ca997a64c858f3867cc 2024-11-25T05:56:51,709 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/1d7af0ce5cab4b7e9fc2120035701417, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/f7bde89c7cd641fb81ce79f7813b59a2, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/8b74f235df864c4daa15470f9cb725ab, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/6e8a2ee8ddde44798964b44d420d324d, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/92e5f727b6c7409da4fe05d8c4b4e108, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/070c5d7fae994adb936dc8f4ea584169, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/e8e27d4174684e86b0729544f21c1cab, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/14fa66276ef448368e5a3c15684c5fcb, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/4e49c0474658488ba552371b183902ef, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/5ab476f341f54c188b704aa937b397c4, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/385cbe658682415db4c1e10869caae6b, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/e53f3fd6372c42b79d85f4ce8eadb0d9, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/6d7a679a3b6f4de4b2f80d47a1584f47, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/9973f95f03184439919ce1b3c09e2d81, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/a7ffbfc31b724db3892db08cdf2f635a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/341f0ec12071448dbab214f119c0f81e, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/48d88adad3b146828e1e743b929989e0, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/91dcc0cf8ea444d0b67043a306813ad7, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/58cbff72752d482da8c669a956313736] to archive 2024-11-25T05:56:51,710 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T05:56:51,712 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/1d7af0ce5cab4b7e9fc2120035701417 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/1d7af0ce5cab4b7e9fc2120035701417 2024-11-25T05:56:51,714 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/f7bde89c7cd641fb81ce79f7813b59a2 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/f7bde89c7cd641fb81ce79f7813b59a2 2024-11-25T05:56:51,715 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/8b74f235df864c4daa15470f9cb725ab to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/8b74f235df864c4daa15470f9cb725ab 2024-11-25T05:56:51,716 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/6e8a2ee8ddde44798964b44d420d324d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/6e8a2ee8ddde44798964b44d420d324d 2024-11-25T05:56:51,717 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/92e5f727b6c7409da4fe05d8c4b4e108 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/92e5f727b6c7409da4fe05d8c4b4e108 2024-11-25T05:56:51,718 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/070c5d7fae994adb936dc8f4ea584169 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/070c5d7fae994adb936dc8f4ea584169 2024-11-25T05:56:51,719 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/e8e27d4174684e86b0729544f21c1cab to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/e8e27d4174684e86b0729544f21c1cab 2024-11-25T05:56:51,720 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/14fa66276ef448368e5a3c15684c5fcb to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/14fa66276ef448368e5a3c15684c5fcb 2024-11-25T05:56:51,721 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/4e49c0474658488ba552371b183902ef to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/4e49c0474658488ba552371b183902ef 2024-11-25T05:56:51,722 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/5ab476f341f54c188b704aa937b397c4 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/5ab476f341f54c188b704aa937b397c4 2024-11-25T05:56:51,723 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/385cbe658682415db4c1e10869caae6b to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/385cbe658682415db4c1e10869caae6b 2024-11-25T05:56:51,724 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/e53f3fd6372c42b79d85f4ce8eadb0d9 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/e53f3fd6372c42b79d85f4ce8eadb0d9 2024-11-25T05:56:51,726 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/6d7a679a3b6f4de4b2f80d47a1584f47 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/6d7a679a3b6f4de4b2f80d47a1584f47 2024-11-25T05:56:51,727 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/9973f95f03184439919ce1b3c09e2d81 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/9973f95f03184439919ce1b3c09e2d81 2024-11-25T05:56:51,728 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/a7ffbfc31b724db3892db08cdf2f635a to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/a7ffbfc31b724db3892db08cdf2f635a 2024-11-25T05:56:51,729 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/341f0ec12071448dbab214f119c0f81e to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/341f0ec12071448dbab214f119c0f81e 2024-11-25T05:56:51,731 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/48d88adad3b146828e1e743b929989e0 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/48d88adad3b146828e1e743b929989e0 2024-11-25T05:56:51,732 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/91dcc0cf8ea444d0b67043a306813ad7 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/91dcc0cf8ea444d0b67043a306813ad7 2024-11-25T05:56:51,733 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/58cbff72752d482da8c669a956313736 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/58cbff72752d482da8c669a956313736 2024-11-25T05:56:51,737 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/5b87bc3d74ec409e9fe5d4f2a4bc195b, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/add9b508983046dbace60a552116a048, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/4d5a7c71b9654d67ac708492a40095a5, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/698c56d0f6594c7c8eb3cde6a547678f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/835e4a2fb0594e2eb1d045a220946e75, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/c2d074166f714f058ad41c4a6c39f5ad, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/50db4314e1ef4a74b84b4afb78fd9822, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/250c260f8cb64b1986d19a8c001453d7, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/0b12471b2abe45b7a3ba9bd1b92cf5f7, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/c89b61141f1b4e41b5d01a198b516938, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/5725dd2bdd9e4fcfa14ebd47f816ef38, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/2b031c403164448daeb3f61a048ca7ac, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/75005d15561a4356aee6ac86e885ebe6, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/f0bbab63ea0846c3b499779adb251b59, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/3b417e60f7d94035a548c79eb442dc81, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/a2f6b0b961894f1d96a91f2eeb60d58f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/c3b1c81579724ed2a3044351bc6ee339, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/d75b91e661fd4cbfa0284941ab9955af, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/e926848829124d78ac19358814a4ad0b] to archive 2024-11-25T05:56:51,738 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T05:56:51,739 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/5b87bc3d74ec409e9fe5d4f2a4bc195b to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/5b87bc3d74ec409e9fe5d4f2a4bc195b 2024-11-25T05:56:51,741 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/add9b508983046dbace60a552116a048 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/add9b508983046dbace60a552116a048 2024-11-25T05:56:51,742 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/4d5a7c71b9654d67ac708492a40095a5 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/4d5a7c71b9654d67ac708492a40095a5 2024-11-25T05:56:51,743 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/698c56d0f6594c7c8eb3cde6a547678f to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/698c56d0f6594c7c8eb3cde6a547678f 2024-11-25T05:56:51,744 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/835e4a2fb0594e2eb1d045a220946e75 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/835e4a2fb0594e2eb1d045a220946e75 2024-11-25T05:56:51,745 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/c2d074166f714f058ad41c4a6c39f5ad to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/c2d074166f714f058ad41c4a6c39f5ad 2024-11-25T05:56:51,747 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/50db4314e1ef4a74b84b4afb78fd9822 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/50db4314e1ef4a74b84b4afb78fd9822 2024-11-25T05:56:51,748 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/250c260f8cb64b1986d19a8c001453d7 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/250c260f8cb64b1986d19a8c001453d7 2024-11-25T05:56:51,750 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/0b12471b2abe45b7a3ba9bd1b92cf5f7 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/0b12471b2abe45b7a3ba9bd1b92cf5f7 2024-11-25T05:56:51,751 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/c89b61141f1b4e41b5d01a198b516938 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/c89b61141f1b4e41b5d01a198b516938 2024-11-25T05:56:51,752 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/5725dd2bdd9e4fcfa14ebd47f816ef38 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/5725dd2bdd9e4fcfa14ebd47f816ef38 2024-11-25T05:56:51,754 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/2b031c403164448daeb3f61a048ca7ac to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/2b031c403164448daeb3f61a048ca7ac 2024-11-25T05:56:51,755 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/75005d15561a4356aee6ac86e885ebe6 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/75005d15561a4356aee6ac86e885ebe6 2024-11-25T05:56:51,757 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/f0bbab63ea0846c3b499779adb251b59 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/f0bbab63ea0846c3b499779adb251b59 2024-11-25T05:56:51,758 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/3b417e60f7d94035a548c79eb442dc81 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/3b417e60f7d94035a548c79eb442dc81 2024-11-25T05:56:51,759 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/a2f6b0b961894f1d96a91f2eeb60d58f to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/a2f6b0b961894f1d96a91f2eeb60d58f 2024-11-25T05:56:51,760 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/c3b1c81579724ed2a3044351bc6ee339 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/c3b1c81579724ed2a3044351bc6ee339 2024-11-25T05:56:51,761 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/d75b91e661fd4cbfa0284941ab9955af to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/d75b91e661fd4cbfa0284941ab9955af 2024-11-25T05:56:51,762 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8ef925b832e3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/e926848829124d78ac19358814a4ad0b to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/e926848829124d78ac19358814a4ad0b 2024-11-25T05:56:52,667 INFO [Thread-1620 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1620. 2024-11-25T05:56:52,667 DEBUG [Thread-1620 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:56:52,667 DEBUG [Thread-1620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:52,667 DEBUG [Thread-1620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:52,667 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:56:55,397 INFO [Thread-1616 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1616. 2024-11-25T05:56:55,397 DEBUG [Thread-1616 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:56:55,397 DEBUG [Thread-1616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:55,397 DEBUG [Thread-1616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:55,397 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:56:55,436 INFO [Thread-1614 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1614. 2024-11-25T05:56:55,436 DEBUG [Thread-1614 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:56:55,436 DEBUG [Thread-1614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:55,437 DEBUG [Thread-1614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:55,437 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:56:55,437 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-25T05:56:55,437 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-25T05:56:55,437 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-11-25T05:56:55,437 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-25T05:56:55,437 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-11-25T05:56:55,437 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-25T05:56:55,437 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-25T05:56:55,437 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-25T05:56:55,438 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1065 2024-11-25T05:56:55,438 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3188 rows 2024-11-25T05:56:55,438 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1067 2024-11-25T05:56:55,438 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3191 rows 2024-11-25T05:56:55,438 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1069 2024-11-25T05:56:55,438 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3199 rows 2024-11-25T05:56:55,438 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1057 2024-11-25T05:56:55,438 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3171 rows 2024-11-25T05:56:55,438 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1068 2024-11-25T05:56:55,438 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3197 rows 2024-11-25T05:56:55,438 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T05:56:55,438 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T05:56:55,438 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:55,439 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:55,439 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:56:55,440 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-25T05:56:55,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-25T05:56:55,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-25T05:56:55,446 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732514215446"}]},"ts":"1732514215446"} 2024-11-25T05:56:55,448 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-25T05:56:55,448 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-25T05:56:55,448 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T05:56:55,450 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef2460dfb8241e76bc050e18474be77d, UNASSIGN}] 2024-11-25T05:56:55,450 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef2460dfb8241e76bc050e18474be77d, UNASSIGN 2024-11-25T05:56:55,451 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=103 updating hbase:meta row=ef2460dfb8241e76bc050e18474be77d, regionState=CLOSING, regionLocation=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:55,452 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef2460dfb8241e76bc050e18474be77d, UNASSIGN because future has completed 2024-11-25T05:56:55,452 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T05:56:55,452 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; CloseRegionProcedure ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005}] 2024-11-25T05:56:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-25T05:56:55,605 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(122): Close ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:55,606 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T05:56:55,606 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1722): Closing ef2460dfb8241e76bc050e18474be77d, disabling compactions & flushes 2024-11-25T05:56:55,606 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:55,606 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:55,606 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. after waiting 0 ms 2024-11-25T05:56:55,606 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:55,606 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(2902): Flushing ef2460dfb8241e76bc050e18474be77d 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-25T05:56:55,607 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=A 2024-11-25T05:56:55,607 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:55,607 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=B 2024-11-25T05:56:55,607 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:55,607 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ef2460dfb8241e76bc050e18474be77d, store=C 2024-11-25T05:56:55,608 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:55,618 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125b87d4c2ebb4c45d2a2d898d7b67ffa94_ef2460dfb8241e76bc050e18474be77d is 50, key is test_row_1/A:col10/1732514215433/Put/seqid=0 2024-11-25T05:56:55,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742266_1442 (size=9914) 2024-11-25T05:56:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-25T05:56:56,025 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:56:56,034 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125b87d4c2ebb4c45d2a2d898d7b67ffa94_ef2460dfb8241e76bc050e18474be77d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125b87d4c2ebb4c45d2a2d898d7b67ffa94_ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:56,035 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/690aed013b8e4a1c9c6e72a8d7473573, store: [table=TestAcidGuarantees family=A region=ef2460dfb8241e76bc050e18474be77d] 2024-11-25T05:56:56,036 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/690aed013b8e4a1c9c6e72a8d7473573 is 175, key is test_row_1/A:col10/1732514215433/Put/seqid=0 2024-11-25T05:56:56,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742267_1443 (size=22561) 2024-11-25T05:56:56,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-25T05:56:56,443 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=349, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/690aed013b8e4a1c9c6e72a8d7473573 2024-11-25T05:56:56,453 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/6c74adc8f77942da80ce556f6fda1258 is 50, key is test_row_1/B:col10/1732514215433/Put/seqid=0 2024-11-25T05:56:56,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742268_1444 (size=9857) 2024-11-25T05:56:56,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-25T05:56:56,859 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/6c74adc8f77942da80ce556f6fda1258 2024-11-25T05:56:56,870 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/098a3db0403449dcb6f2708bb6ef5f66 is 50, key is test_row_1/C:col10/1732514215433/Put/seqid=0 2024-11-25T05:56:56,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742269_1445 (size=9857) 2024-11-25T05:56:57,275 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/098a3db0403449dcb6f2708bb6ef5f66 2024-11-25T05:56:57,282 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/A/690aed013b8e4a1c9c6e72a8d7473573 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/690aed013b8e4a1c9c6e72a8d7473573 2024-11-25T05:56:57,285 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/690aed013b8e4a1c9c6e72a8d7473573, entries=100, sequenceid=349, filesize=22.0 K 2024-11-25T05:56:57,286 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/B/6c74adc8f77942da80ce556f6fda1258 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/6c74adc8f77942da80ce556f6fda1258 2024-11-25T05:56:57,289 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/6c74adc8f77942da80ce556f6fda1258, entries=100, sequenceid=349, filesize=9.6 K 2024-11-25T05:56:57,290 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/.tmp/C/098a3db0403449dcb6f2708bb6ef5f66 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/098a3db0403449dcb6f2708bb6ef5f66 2024-11-25T05:56:57,293 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/098a3db0403449dcb6f2708bb6ef5f66, entries=100, sequenceid=349, filesize=9.6 K 2024-11-25T05:56:57,293 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for ef2460dfb8241e76bc050e18474be77d in 1687ms, sequenceid=349, compaction requested=true 2024-11-25T05:56:57,297 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/recovered.edits/352.seqid, newMaxSeqId=352, maxSeqId=4 2024-11-25T05:56:57,297 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. 2024-11-25T05:56:57,297 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1676): Region close journal for ef2460dfb8241e76bc050e18474be77d: Waiting for close lock at 1732514215606Running coprocessor pre-close hooks at 1732514215606Disabling compacts and flushes for region at 1732514215606Disabling writes for close at 1732514215606Obtaining lock to block concurrent updates at 1732514215606Preparing flush snapshotting stores in ef2460dfb8241e76bc050e18474be77d at 1732514215607 (+1 ms)Finished memstore snapshotting TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732514215608 (+1 ms)Flushing stores of TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d. at 1732514215609 (+1 ms)Flushing ef2460dfb8241e76bc050e18474be77d/A: creating writer at 1732514215609Flushing mob file ef2460dfb8241e76bc050e18474be77d/A: appending metadata at 1732514215618 (+9 ms)Flushing mob file ef2460dfb8241e76bc050e18474be77d/A: closing flushed file at 1732514215618Flushing ef2460dfb8241e76bc050e18474be77d/A: appending metadata at 1732514216035 (+417 ms)Flushing ef2460dfb8241e76bc050e18474be77d/A: closing flushed file at 1732514216035Flushing ef2460dfb8241e76bc050e18474be77d/B: creating writer at 1732514216450 (+415 ms)Flushing ef2460dfb8241e76bc050e18474be77d/B: appending metadata at 1732514216453 (+3 ms)Flushing ef2460dfb8241e76bc050e18474be77d/B: closing flushed file at 1732514216453Flushing ef2460dfb8241e76bc050e18474be77d/C: creating writer at 1732514216866 (+413 ms)Flushing ef2460dfb8241e76bc050e18474be77d/C: appending metadata at 1732514216869 (+3 ms)Flushing ef2460dfb8241e76bc050e18474be77d/C: closing flushed file at 1732514216869Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40bdc7c5: reopening flushed file at 1732514217281 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@362afbab: reopening flushed file at 1732514217285 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c044a08: reopening flushed file at 1732514217289 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for ef2460dfb8241e76bc050e18474be77d in 1687ms, sequenceid=349, compaction requested=true at 1732514217293 (+4 ms)Writing region close event to WAL at 1732514217294 (+1 ms)Running coprocessor post-close hooks at 1732514217297 (+3 ms)Closed at 1732514217297 2024-11-25T05:56:57,299 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(157): Closed ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:57,299 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=103 updating hbase:meta row=ef2460dfb8241e76bc050e18474be77d, regionState=CLOSED 2024-11-25T05:56:57,300 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=103, state=RUNNABLE, hasLock=false; CloseRegionProcedure ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 because future has completed 2024-11-25T05:56:57,302 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-25T05:56:57,302 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; CloseRegionProcedure ef2460dfb8241e76bc050e18474be77d, server=8ef925b832e3,35599,1732514089005 in 1.8490 sec 2024-11-25T05:56:57,303 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=103, resume processing ppid=102 2024-11-25T05:56:57,303 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, ppid=102, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef2460dfb8241e76bc050e18474be77d, UNASSIGN in 1.8520 sec 2024-11-25T05:56:57,305 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-25T05:56:57,305 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8560 sec 2024-11-25T05:56:57,306 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732514217306"}]},"ts":"1732514217306"} 2024-11-25T05:56:57,307 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-25T05:56:57,307 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-25T05:56:57,308 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8660 sec 2024-11-25T05:56:57,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-25T05:56:57,604 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:56:57,605 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-25T05:56:57,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T05:56:57,609 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=105, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T05:56:57,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-25T05:56:57,611 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=105, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T05:56:57,614 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:57,616 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A, FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B, FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C, FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/recovered.edits] 2024-11-25T05:56:57,620 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/296e0a112ac5451990b56a7092c32af3 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/296e0a112ac5451990b56a7092c32af3 2024-11-25T05:56:57,622 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/30c02f3bd6c94256a2857eb18565ef90 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/30c02f3bd6c94256a2857eb18565ef90 2024-11-25T05:56:57,623 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/690aed013b8e4a1c9c6e72a8d7473573 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/A/690aed013b8e4a1c9c6e72a8d7473573 2024-11-25T05:56:57,627 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/0a2f81d1c8344de9850fc4011114855e to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/0a2f81d1c8344de9850fc4011114855e 2024-11-25T05:56:57,628 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/6c74adc8f77942da80ce556f6fda1258 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/6c74adc8f77942da80ce556f6fda1258 2024-11-25T05:56:57,630 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/6cc034a1d36044d7aa7daf6ca43660e3 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/B/6cc034a1d36044d7aa7daf6ca43660e3 2024-11-25T05:56:57,633 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/098a3db0403449dcb6f2708bb6ef5f66 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/098a3db0403449dcb6f2708bb6ef5f66 2024-11-25T05:56:57,634 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/43760394cbc04f6bb9ac2ae805f4cd4e to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/43760394cbc04f6bb9ac2ae805f4cd4e 2024-11-25T05:56:57,636 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/5995e1a8b8774c1ba5260a653b9e3a3b to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/C/5995e1a8b8774c1ba5260a653b9e3a3b 2024-11-25T05:56:57,640 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/recovered.edits/352.seqid to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d/recovered.edits/352.seqid 2024-11-25T05:56:57,641 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:57,641 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-25T05:56:57,642 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-25T05:56:57,643 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-25T05:56:57,647 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112556b41fdb7335446e8f5c5c3bf51da7f0_ef2460dfb8241e76bc050e18474be77d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112556b41fdb7335446e8f5c5c3bf51da7f0_ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:57,649 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112558eff27fae5b40b9a7b1090ab22bf4bd_ef2460dfb8241e76bc050e18474be77d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112558eff27fae5b40b9a7b1090ab22bf4bd_ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:57,650 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411255d58c8e3efe040f2acc3211cc10ecb4d_ef2460dfb8241e76bc050e18474be77d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411255d58c8e3efe040f2acc3211cc10ecb4d_ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:57,650 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112562780183490b416e8afee077f0a599ea_ef2460dfb8241e76bc050e18474be77d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112562780183490b416e8afee077f0a599ea_ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:57,651 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112564f0f5a04f2647ea83c521f2a36b3826_ef2460dfb8241e76bc050e18474be77d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112564f0f5a04f2647ea83c521f2a36b3826_ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:57,652 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125791259ecd65a48bd8efe8969f38ed277_ef2460dfb8241e76bc050e18474be77d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125791259ecd65a48bd8efe8969f38ed277_ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:57,653 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125a1b396fafc23460aaa2bac920504a510_ef2460dfb8241e76bc050e18474be77d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125a1b396fafc23460aaa2bac920504a510_ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:57,653 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125a280c2e976724bb9ae2824ddeb0378f0_ef2460dfb8241e76bc050e18474be77d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125a280c2e976724bb9ae2824ddeb0378f0_ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:57,654 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125a3f1a4d4f6e44151b419d2e9289b9809_ef2460dfb8241e76bc050e18474be77d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125a3f1a4d4f6e44151b419d2e9289b9809_ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:57,655 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125b87d4c2ebb4c45d2a2d898d7b67ffa94_ef2460dfb8241e76bc050e18474be77d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125b87d4c2ebb4c45d2a2d898d7b67ffa94_ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:57,655 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125bb8d8231f030497d955e5690227eb854_ef2460dfb8241e76bc050e18474be77d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125bb8d8231f030497d955e5690227eb854_ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:57,656 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125bd89193fb0704af3b0268e6b28d8be24_ef2460dfb8241e76bc050e18474be77d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125bd89193fb0704af3b0268e6b28d8be24_ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:57,657 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c14d19322bf74ad3a5d7e39ff6e993cf_ef2460dfb8241e76bc050e18474be77d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c14d19322bf74ad3a5d7e39ff6e993cf_ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:57,658 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c2ff5642bbed41b580272969c76122b2_ef2460dfb8241e76bc050e18474be77d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c2ff5642bbed41b580272969c76122b2_ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:57,658 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d2d65cd494414891834e9e6689527e60_ef2460dfb8241e76bc050e18474be77d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d2d65cd494414891834e9e6689527e60_ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:57,659 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125e9d6dfdcecc244ee83b7a64822f4a330_ef2460dfb8241e76bc050e18474be77d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125e9d6dfdcecc244ee83b7a64822f4a330_ef2460dfb8241e76bc050e18474be77d 2024-11-25T05:56:57,659 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-25T05:56:57,661 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=105, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T05:56:57,663 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-25T05:56:57,664 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-25T05:56:57,665 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=105, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T05:56:57,665 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-25T05:56:57,665 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732514217665"}]},"ts":"9223372036854775807"} 2024-11-25T05:56:57,667 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-25T05:56:57,667 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ef2460dfb8241e76bc050e18474be77d, NAME => 'TestAcidGuarantees,,1732514181439.ef2460dfb8241e76bc050e18474be77d.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T05:56:57,667 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-25T05:56:57,667 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732514217667"}]},"ts":"9223372036854775807"} 2024-11-25T05:56:57,669 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-25T05:56:57,669 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=105, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T05:56:57,670 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 64 msec 2024-11-25T05:56:57,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-25T05:56:57,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-25T05:56:57,722 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:56:57,730 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=247 (was 247), OpenFileDescriptor=443 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=364 (was 411), ProcessCount=11 (was 11), AvailableMemoryMB=7611 (was 7642) 2024-11-25T05:56:57,738 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=247, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=364, ProcessCount=11, AvailableMemoryMB=7611 2024-11-25T05:56:57,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T05:56:57,739 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T05:56:57,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=106, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-25T05:56:57,741 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T05:56:57,741 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:56:57,741 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 106 2024-11-25T05:56:57,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-25T05:56:57,742 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T05:56:57,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742270_1446 (size=1061) 2024-11-25T05:56:57,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-25T05:56:58,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-25T05:56:58,148 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a 2024-11-25T05:56:58,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742271_1447 (size=53) 2024-11-25T05:56:58,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-25T05:56:58,556 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T05:56:58,556 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 39bebe481dd46e37e6dae001fcad1e53, disabling compactions & flushes 2024-11-25T05:56:58,556 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:56:58,556 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:56:58,556 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. after waiting 0 ms 2024-11-25T05:56:58,556 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:56:58,556 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:56:58,556 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 39bebe481dd46e37e6dae001fcad1e53: Waiting for close lock at 1732514218556Disabling compacts and flushes for region at 1732514218556Disabling writes for close at 1732514218556Writing region close event to WAL at 1732514218556Closed at 1732514218556 2024-11-25T05:56:58,559 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T05:56:58,559 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732514218559"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732514218559"}]},"ts":"1732514218559"} 2024-11-25T05:56:58,563 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T05:56:58,565 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T05:56:58,566 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732514218565"}]},"ts":"1732514218565"} 2024-11-25T05:56:58,568 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-25T05:56:58,568 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=39bebe481dd46e37e6dae001fcad1e53, ASSIGN}] 2024-11-25T05:56:58,569 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=39bebe481dd46e37e6dae001fcad1e53, ASSIGN 2024-11-25T05:56:58,570 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=39bebe481dd46e37e6dae001fcad1e53, ASSIGN; state=OFFLINE, location=8ef925b832e3,35599,1732514089005; forceNewPlan=false, retain=false 2024-11-25T05:56:58,722 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=39bebe481dd46e37e6dae001fcad1e53, regionState=OPENING, regionLocation=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:58,725 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=39bebe481dd46e37e6dae001fcad1e53, ASSIGN because future has completed 2024-11-25T05:56:58,725 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; OpenRegionProcedure 39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005}] 2024-11-25T05:56:58,765 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-25T05:56:58,881 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:56:58,881 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7752): Opening region: {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} 2024-11-25T05:56:58,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-25T05:56:58,883 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 39bebe481dd46e37e6dae001fcad1e53 2024-11-25T05:56:58,884 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T05:56:58,884 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7794): checking encryption for 39bebe481dd46e37e6dae001fcad1e53 2024-11-25T05:56:58,884 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7797): checking classloading for 39bebe481dd46e37e6dae001fcad1e53 2024-11-25T05:56:58,886 INFO [StoreOpener-39bebe481dd46e37e6dae001fcad1e53-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 39bebe481dd46e37e6dae001fcad1e53 2024-11-25T05:56:58,888 INFO [StoreOpener-39bebe481dd46e37e6dae001fcad1e53-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T05:56:58,889 INFO [StoreOpener-39bebe481dd46e37e6dae001fcad1e53-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 39bebe481dd46e37e6dae001fcad1e53 columnFamilyName A 2024-11-25T05:56:58,889 DEBUG [StoreOpener-39bebe481dd46e37e6dae001fcad1e53-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:56:58,890 INFO [StoreOpener-39bebe481dd46e37e6dae001fcad1e53-1 {}] regionserver.HStore(327): Store=39bebe481dd46e37e6dae001fcad1e53/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T05:56:58,890 INFO [StoreOpener-39bebe481dd46e37e6dae001fcad1e53-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 39bebe481dd46e37e6dae001fcad1e53 2024-11-25T05:56:58,892 INFO [StoreOpener-39bebe481dd46e37e6dae001fcad1e53-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T05:56:58,892 INFO [StoreOpener-39bebe481dd46e37e6dae001fcad1e53-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 39bebe481dd46e37e6dae001fcad1e53 columnFamilyName B 2024-11-25T05:56:58,893 DEBUG [StoreOpener-39bebe481dd46e37e6dae001fcad1e53-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:56:58,893 INFO [StoreOpener-39bebe481dd46e37e6dae001fcad1e53-1 {}] regionserver.HStore(327): Store=39bebe481dd46e37e6dae001fcad1e53/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T05:56:58,893 INFO [StoreOpener-39bebe481dd46e37e6dae001fcad1e53-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 39bebe481dd46e37e6dae001fcad1e53 2024-11-25T05:56:58,894 INFO [StoreOpener-39bebe481dd46e37e6dae001fcad1e53-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T05:56:58,895 INFO [StoreOpener-39bebe481dd46e37e6dae001fcad1e53-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 39bebe481dd46e37e6dae001fcad1e53 columnFamilyName C 2024-11-25T05:56:58,895 DEBUG [StoreOpener-39bebe481dd46e37e6dae001fcad1e53-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:56:58,895 INFO [StoreOpener-39bebe481dd46e37e6dae001fcad1e53-1 {}] regionserver.HStore(327): Store=39bebe481dd46e37e6dae001fcad1e53/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T05:56:58,896 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:56:58,896 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1038): replaying wal for 39bebe481dd46e37e6dae001fcad1e53 2024-11-25T05:56:58,896 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53 2024-11-25T05:56:58,897 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53 2024-11-25T05:56:58,898 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1048): stopping wal replay for 39bebe481dd46e37e6dae001fcad1e53 2024-11-25T05:56:58,898 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1060): Cleaning up temporary data for 39bebe481dd46e37e6dae001fcad1e53 2024-11-25T05:56:58,899 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T05:56:58,900 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1093): writing seq id for 39bebe481dd46e37e6dae001fcad1e53 2024-11-25T05:56:58,902 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T05:56:58,902 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1114): Opened 39bebe481dd46e37e6dae001fcad1e53; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62709817, jitterRate=-0.06555090844631195}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T05:56:58,902 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 39bebe481dd46e37e6dae001fcad1e53 2024-11-25T05:56:58,903 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1006): Region open journal for 39bebe481dd46e37e6dae001fcad1e53: Running coprocessor pre-open hook at 1732514218884Writing region info on filesystem at 1732514218884Initializing all the Stores at 1732514218885 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732514218885Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732514218886 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732514218886Cleaning up temporary data from old regions at 1732514218898 (+12 ms)Running coprocessor post-open hooks at 1732514218902 (+4 ms)Region opened successfully at 1732514218903 (+1 ms) 2024-11-25T05:56:58,903 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., pid=108, masterSystemTime=1732514218877 2024-11-25T05:56:58,905 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:56:58,905 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:56:58,905 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=39bebe481dd46e37e6dae001fcad1e53, regionState=OPEN, openSeqNum=2, regionLocation=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:58,906 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=108, ppid=107, state=RUNNABLE, hasLock=false; OpenRegionProcedure 39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 because future has completed 2024-11-25T05:56:58,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-25T05:56:58,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; OpenRegionProcedure 39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 in 182 msec 2024-11-25T05:56:58,910 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=107, resume processing ppid=106 2024-11-25T05:56:58,910 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, ppid=106, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=39bebe481dd46e37e6dae001fcad1e53, ASSIGN in 340 msec 2024-11-25T05:56:58,910 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T05:56:58,910 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732514218910"}]},"ts":"1732514218910"} 2024-11-25T05:56:58,911 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-25T05:56:58,912 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T05:56:58,913 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1730 sec 2024-11-25T05:56:59,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-25T05:56:59,894 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:56:59,896 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dacd8f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:59,897 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:56:59,897 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:56:59,900 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:56:59,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:56:59,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:56:59,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e1f592a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:59,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:56:59,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:56:59,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:59,902 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40678, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:56:59,903 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3de78ccf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:59,904 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:56:59,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:56:59,905 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:56:59,906 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34962, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:56:59,906 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T05:56:59,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 8ef925b832e3,41285,1732514088402 2024-11-25T05:56:59,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@24b5bb6 2024-11-25T05:56:59,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T05:56:59,908 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40682, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T05:56:59,910 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72240ede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:59,910 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:56:59,910 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:56:59,911 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:56:59,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:56:59,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:56:59,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@427d243, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:59,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:56:59,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:56:59,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:59,912 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40690, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:56:59,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bf187cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:59,914 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d6d1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:59,914 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:56:59,914 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:56:59,915 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:56:59,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:56:59,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:56:59,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46f4c800, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:59,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:56:59,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:56:59,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:59,917 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40700, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:56:59,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34785f7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:59,919 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@503e1f3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:59,919 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:56:59,919 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:56:59,920 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:56:59,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:56:59,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:56:59,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2966b24b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:59,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:56:59,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:56:59,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:59,922 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40722, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:56:59,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f164594, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:59,923 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205b33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:59,924 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:56:59,924 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:56:59,924 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:56:59,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:56:59,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:56:59,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f2eb38a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:59,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:56:59,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:56:59,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:59,926 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40736, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:56:59,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@268e01ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:59,927 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e100fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:59,927 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:56:59,927 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:56:59,928 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:56:59,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:56:59,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:56:59,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6438f6ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:59,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:56:59,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:56:59,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:59,929 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40766, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:56:59,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a5f113d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:59,930 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc361f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:59,930 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:56:59,930 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:56:59,931 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:56:59,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:56:59,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:56:59,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4aa42991, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:59,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:56:59,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:56:59,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:59,932 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40778, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:56:59,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c131c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:59,933 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259a56de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:59,933 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:56:59,934 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:56:59,934 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:56:59,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:56:59,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:56:59,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f22b742, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:59,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:56:59,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:56:59,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:59,936 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40798, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:56:59,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@151af7d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:59,937 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4cbf6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:59,937 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:56:59,937 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:56:59,937 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:56:59,937 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:56:59,937 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:56:59,938 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@454b1e5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:59,938 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:56:59,938 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:56:59,938 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:59,939 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40808, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:56:59,939 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f4b3562, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:59,940 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c83334c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:59,940 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:56:59,940 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:56:59,940 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:56:59,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:56:59,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:56:59,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dd538b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:59,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:56:59,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:56:59,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:59,942 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40814, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:56:59,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cfded05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:59,943 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3532b5c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:59,943 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:56:59,943 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:56:59,943 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:56:59,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:56:59,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:56:59,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b6ebdd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:59,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:56:59,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:56:59,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:56:59,944 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:56:59,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@425bedcd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:56:59,945 DEBUG [Thread-2019 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:56:59,946 DEBUG [Thread-2004 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:56:59,946 DEBUG [Thread-2015 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:56:59,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:56:59,946 DEBUG [Thread-2006 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:56:59,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:56:59,947 DEBUG [Thread-2008 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:56:59,947 DEBUG [Thread-2000 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:56:59,947 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:56:59,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:56:59,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:56:59,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:56:59,947 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:56:59,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:56:59,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:56:59,947 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34968, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:56:59,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:56:59,948 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34974, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:56:59,948 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34982, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:56:59,948 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:56:59,948 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34976, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:56:59,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:56:59,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2] 2024-11-25T05:56:59,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2] 2024-11-25T05:56:59,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:56:59,949 DEBUG [Thread-2002 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:56:59,950 DEBUG [Thread-2013 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:56:59,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:56:59,950 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35010, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:56:59,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2] 2024-11-25T05:56:59,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:56:59,950 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:56:59,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2] 2024-11-25T05:56:59,951 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:56:59,951 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:56:59,951 DEBUG [Thread-2011 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:56:59,951 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35022, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:56:59,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2] 2024-11-25T05:56:59,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2] 2024-11-25T05:56:59,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:56:59,952 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35032, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:56:59,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:56:59,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-25T05:56:59,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2] 2024-11-25T05:56:59,953 DEBUG [Thread-2017 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:56:59,953 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:56:59,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2] 2024-11-25T05:56:59,955 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:56:59,955 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2] 2024-11-25T05:56:59,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-25T05:56:59,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:56:59,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:56:59,957 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35058, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:56:59,957 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:56:59,957 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:56:59,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2] 2024-11-25T05:56:59,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on 39bebe481dd46e37e6dae001fcad1e53 2024-11-25T05:56:59,961 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 39bebe481dd46e37e6dae001fcad1e53 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-25T05:56:59,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=A 2024-11-25T05:56:59,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:59,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=B 2024-11-25T05:56:59,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:59,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=C 2024-11-25T05:56:59,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:56:59,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/c8281def41da4402b0b038bcaaddb1f6 is 50, key is test_row_0/A:col10/1732514219954/Put/seqid=0 2024-11-25T05:56:59,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:56:59,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514279989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:56:59,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:59,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:56:59,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:56:59,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732514219982, max timestamp=1732514219993Num uniques 859; 2024-11-25T05:56:59,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732514219982, max timestamp=1732514219993Num uniques 859; 2024-11-25T05:56:59,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732514219982, max timestamp=1732514219993Num uniques 859; 2024-11-25T05:56:59,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-25T05:56:59,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-25T05:56:59,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-25T05:56:59,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742272_1448 (size=14341) 2024-11-25T05:56:59,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/c8281def41da4402b0b038bcaaddb1f6 2024-11-25T05:57:00,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/fbe09a23208b4bc2aa126d7b9c36a238 is 50, key is test_row_0/B:col10/1732514219954/Put/seqid=0 2024-11-25T05:57:00,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742273_1449 (size=12001) 2024-11-25T05:57:00,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:00,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514280021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:00,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:00,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:00,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34996 deadline: 1732514280028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:00,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:00,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:00,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35022 deadline: 1732514280029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:00,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:00,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:00,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34982 deadline: 1732514280030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:00,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:00,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-25T05:57:00,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:00,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514280102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:00,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:00,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-25T05:57:00,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:00,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:00,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:00,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:00,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:00,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:00,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:00,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514280132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:00,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:00,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:00,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34996 deadline: 1732514280142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:00,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:00,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35022 deadline: 1732514280142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:00,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:00,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:00,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:00,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34982 deadline: 1732514280152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:00,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:00,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-25T05:57:00,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:00,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:00,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:00,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:00,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:00,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:00,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-25T05:57:00,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:00,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514280312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:00,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:00,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:00,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514280342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:00,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:00,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:00,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34996 deadline: 1732514280362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:00,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:00,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35022 deadline: 1732514280362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:00,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:00,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:00,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34982 deadline: 1732514280362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:00,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:00,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:00,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-25T05:57:00,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:00,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:00,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:00,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:00,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:00,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:00,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/fbe09a23208b4bc2aa126d7b9c36a238 2024-11-25T05:57:00,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/4ff6f500204c4769bcddc48c412a3115 is 50, key is test_row_0/C:col10/1732514219954/Put/seqid=0 2024-11-25T05:57:00,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742274_1450 (size=12001) 2024-11-25T05:57:00,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-25T05:57:00,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:00,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:00,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:00,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:00,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:00,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:00,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-25T05:57:00,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:00,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514280632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:00,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:00,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:00,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514280652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:00,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:00,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34996 deadline: 1732514280672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:00,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:00,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35022 deadline: 1732514280672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:00,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:00,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:00,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34982 deadline: 1732514280672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:00,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:00,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:00,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-25T05:57:00,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:00,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:00,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:00,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:00,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:00,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/4ff6f500204c4769bcddc48c412a3115 2024-11-25T05:57:00,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/c8281def41da4402b0b038bcaaddb1f6 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/c8281def41da4402b0b038bcaaddb1f6 2024-11-25T05:57:00,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/c8281def41da4402b0b038bcaaddb1f6, entries=200, sequenceid=12, filesize=14.0 K 2024-11-25T05:57:00,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/fbe09a23208b4bc2aa126d7b9c36a238 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/fbe09a23208b4bc2aa126d7b9c36a238 2024-11-25T05:57:00,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/fbe09a23208b4bc2aa126d7b9c36a238, entries=150, sequenceid=12, filesize=11.7 K 2024-11-25T05:57:00,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/4ff6f500204c4769bcddc48c412a3115 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/4ff6f500204c4769bcddc48c412a3115 2024-11-25T05:57:00,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/4ff6f500204c4769bcddc48c412a3115, entries=150, sequenceid=12, filesize=11.7 K 2024-11-25T05:57:00,859 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 39bebe481dd46e37e6dae001fcad1e53 in 899ms, sequenceid=12, compaction requested=false 2024-11-25T05:57:00,859 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-25T05:57:00,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:00,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-25T05:57:00,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:00,873 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 39bebe481dd46e37e6dae001fcad1e53 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-25T05:57:00,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=A 2024-11-25T05:57:00,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:57:00,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=B 2024-11-25T05:57:00,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:57:00,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=C 2024-11-25T05:57:00,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:57:00,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/59971285dc6f49e49e2ca4fa13965541 is 50, key is test_row_0/A:col10/1732514220026/Put/seqid=0 2024-11-25T05:57:00,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742275_1451 (size=12001) 2024-11-25T05:57:01,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-25T05:57:01,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on 39bebe481dd46e37e6dae001fcad1e53 2024-11-25T05:57:01,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:01,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:01,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34982 deadline: 1732514281182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:01,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:01,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:01,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:01,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:01,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514281185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:01,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:01,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:01,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:01,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:01,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514281187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:01,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:01,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:01,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:01,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:01,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34996 deadline: 1732514281192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:01,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:01,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:01,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35022 deadline: 1732514281192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:01,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:01,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:01,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:01,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:01,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:01,280 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/59971285dc6f49e49e2ca4fa13965541 2024-11-25T05:57:01,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/ba01b74530b9405188eacce70f28efee is 50, key is test_row_0/B:col10/1732514220026/Put/seqid=0 2024-11-25T05:57:01,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742276_1452 (size=12001) 2024-11-25T05:57:01,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:01,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514281302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:01,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:01,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:01,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:01,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:01,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514281302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:01,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:01,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:01,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:01,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:01,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514281512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:01,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:01,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:01,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:01,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:01,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514281522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:01,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:01,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:01,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:01,692 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/ba01b74530b9405188eacce70f28efee 2024-11-25T05:57:01,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/d866dfc1e036418dad370654f888245f is 50, key is test_row_0/C:col10/1732514220026/Put/seqid=0 2024-11-25T05:57:01,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742277_1453 (size=12001) 2024-11-25T05:57:01,702 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/d866dfc1e036418dad370654f888245f 2024-11-25T05:57:01,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/59971285dc6f49e49e2ca4fa13965541 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/59971285dc6f49e49e2ca4fa13965541 2024-11-25T05:57:01,708 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/59971285dc6f49e49e2ca4fa13965541, entries=150, sequenceid=46, filesize=11.7 K 2024-11-25T05:57:01,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/ba01b74530b9405188eacce70f28efee as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/ba01b74530b9405188eacce70f28efee 2024-11-25T05:57:01,714 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/ba01b74530b9405188eacce70f28efee, entries=150, sequenceid=46, filesize=11.7 K 2024-11-25T05:57:01,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/d866dfc1e036418dad370654f888245f as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/d866dfc1e036418dad370654f888245f 2024-11-25T05:57:01,718 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/d866dfc1e036418dad370654f888245f, entries=150, sequenceid=46, filesize=11.7 K 2024-11-25T05:57:01,719 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for 39bebe481dd46e37e6dae001fcad1e53 in 846ms, sequenceid=46, compaction requested=false 2024-11-25T05:57:01,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:01,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:01,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-25T05:57:01,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-25T05:57:01,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-25T05:57:01,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7630 sec 2024-11-25T05:57:01,723 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.7710 sec 2024-11-25T05:57:01,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on 39bebe481dd46e37e6dae001fcad1e53 2024-11-25T05:57:01,826 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 39bebe481dd46e37e6dae001fcad1e53 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-25T05:57:01,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=A 2024-11-25T05:57:01,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:01,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=B 2024-11-25T05:57:01,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:01,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=C 2024-11-25T05:57:01,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:01,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/682ddbd0a47247eb85bc598865e49c8f is 50, key is test_row_0/A:col10/1732514221825/Put/seqid=0 2024-11-25T05:57:01,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742278_1454 (size=14341) 2024-11-25T05:57:01,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:01,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514281890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:01,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:01,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:01,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:01,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:01,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514281893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:01,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:01,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:01,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:02,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:02,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514282002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:02,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:02,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:02,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:02,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:02,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514282012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:02,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:02,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:02,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:02,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-25T05:57:02,102 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:57:02,102 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:57:02,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-25T05:57:02,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-25T05:57:02,104 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:57:02,104 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:57:02,104 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:57:02,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-25T05:57:02,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:02,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34982 deadline: 1732514282212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:02,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:02,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34996 deadline: 1732514282212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:02,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:02,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35022 deadline: 1732514282212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:02,219 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_2' in 39bebe481dd46e37e6dae001fcad1e53 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:57:02,219 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_2' in 39bebe481dd46e37e6dae001fcad1e53 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:57:02,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:02,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:02,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:02,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:02,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:02,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:02,219 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_1' in 39bebe481dd46e37e6dae001fcad1e53 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:57:02,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:02,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:02,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:02,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:02,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514282222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:02,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:02,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:02,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:02,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/682ddbd0a47247eb85bc598865e49c8f 2024-11-25T05:57:02,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/a4be286995394065923978ff3d0d040b is 50, key is test_row_0/B:col10/1732514221825/Put/seqid=0 2024-11-25T05:57:02,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742279_1455 (size=12001) 2024-11-25T05:57:02,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:02,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514282231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:02,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:02,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:02,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:02,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-25T05:57:02,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:02,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:02,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:02,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:02,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:02,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:02,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-25T05:57:02,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:02,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:02,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:02,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:02,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:02,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:02,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-25T05:57:02,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:02,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514282542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:02,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:02,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:02,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:02,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:02,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514282552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:02,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:02,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:02,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:02,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-25T05:57:02,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:02,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:02,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:02,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:02,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:02,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:02,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/a4be286995394065923978ff3d0d040b 2024-11-25T05:57:02,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/8a11d4a004434a2cb1c46cacde8c0f1a is 50, key is test_row_0/C:col10/1732514221825/Put/seqid=0 2024-11-25T05:57:02,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742280_1456 (size=12001) 2024-11-25T05:57:02,649 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/8a11d4a004434a2cb1c46cacde8c0f1a 2024-11-25T05:57:02,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/682ddbd0a47247eb85bc598865e49c8f as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/682ddbd0a47247eb85bc598865e49c8f 2024-11-25T05:57:02,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/682ddbd0a47247eb85bc598865e49c8f, entries=200, sequenceid=58, filesize=14.0 K 2024-11-25T05:57:02,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/a4be286995394065923978ff3d0d040b as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/a4be286995394065923978ff3d0d040b 2024-11-25T05:57:02,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/a4be286995394065923978ff3d0d040b, entries=150, sequenceid=58, filesize=11.7 K 2024-11-25T05:57:02,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/8a11d4a004434a2cb1c46cacde8c0f1a as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/8a11d4a004434a2cb1c46cacde8c0f1a 2024-11-25T05:57:02,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/8a11d4a004434a2cb1c46cacde8c0f1a, entries=150, sequenceid=58, filesize=11.7 K 2024-11-25T05:57:02,662 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 39bebe481dd46e37e6dae001fcad1e53 in 836ms, sequenceid=58, compaction requested=true 2024-11-25T05:57:02,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:02,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 39bebe481dd46e37e6dae001fcad1e53:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:57:02,662 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:57:02,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:02,662 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:57:02,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 39bebe481dd46e37e6dae001fcad1e53:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:57:02,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:02,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 39bebe481dd46e37e6dae001fcad1e53:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:57:02,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:57:02,663 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:57:02,663 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:57:02,663 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): 39bebe481dd46e37e6dae001fcad1e53/A is initiating minor compaction (all files) 2024-11-25T05:57:02,663 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): 39bebe481dd46e37e6dae001fcad1e53/B is initiating minor compaction (all files) 2024-11-25T05:57:02,663 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 39bebe481dd46e37e6dae001fcad1e53/B in TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:02,663 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 39bebe481dd46e37e6dae001fcad1e53/A in TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:02,663 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/fbe09a23208b4bc2aa126d7b9c36a238, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/ba01b74530b9405188eacce70f28efee, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/a4be286995394065923978ff3d0d040b] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp, totalSize=35.2 K 2024-11-25T05:57:02,663 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/c8281def41da4402b0b038bcaaddb1f6, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/59971285dc6f49e49e2ca4fa13965541, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/682ddbd0a47247eb85bc598865e49c8f] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp, totalSize=39.7 K 2024-11-25T05:57:02,663 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8281def41da4402b0b038bcaaddb1f6, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732514219954 2024-11-25T05:57:02,663 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting fbe09a23208b4bc2aa126d7b9c36a238, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732514219954 2024-11-25T05:57:02,663 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting ba01b74530b9405188eacce70f28efee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732514220019 2024-11-25T05:57:02,663 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59971285dc6f49e49e2ca4fa13965541, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732514220019 2024-11-25T05:57:02,664 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting a4be286995394065923978ff3d0d040b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732514221167 2024-11-25T05:57:02,664 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 682ddbd0a47247eb85bc598865e49c8f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732514221167 2024-11-25T05:57:02,669 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 39bebe481dd46e37e6dae001fcad1e53#A#compaction#386 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:02,669 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/7e984871f62e4f86af09a902842582fa is 50, key is test_row_0/A:col10/1732514221825/Put/seqid=0 2024-11-25T05:57:02,671 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 39bebe481dd46e37e6dae001fcad1e53#B#compaction#387 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:02,671 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/706de24bb95c4636be0d45705f19a869 is 50, key is test_row_0/B:col10/1732514221825/Put/seqid=0 2024-11-25T05:57:02,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742281_1457 (size=12104) 2024-11-25T05:57:02,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742282_1458 (size=12104) 2024-11-25T05:57:02,683 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/7e984871f62e4f86af09a902842582fa as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/7e984871f62e4f86af09a902842582fa 2024-11-25T05:57:02,687 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 39bebe481dd46e37e6dae001fcad1e53/A of 39bebe481dd46e37e6dae001fcad1e53 into 7e984871f62e4f86af09a902842582fa(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:02,687 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:02,687 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., storeName=39bebe481dd46e37e6dae001fcad1e53/A, priority=13, startTime=1732514222662; duration=0sec 2024-11-25T05:57:02,687 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:57:02,687 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 39bebe481dd46e37e6dae001fcad1e53:A 2024-11-25T05:57:02,688 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:57:02,688 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:57:02,688 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): 39bebe481dd46e37e6dae001fcad1e53/C is initiating minor compaction (all files) 2024-11-25T05:57:02,688 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 39bebe481dd46e37e6dae001fcad1e53/C in TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:02,688 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/4ff6f500204c4769bcddc48c412a3115, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/d866dfc1e036418dad370654f888245f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/8a11d4a004434a2cb1c46cacde8c0f1a] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp, totalSize=35.2 K 2024-11-25T05:57:02,689 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ff6f500204c4769bcddc48c412a3115, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732514219954 2024-11-25T05:57:02,689 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting d866dfc1e036418dad370654f888245f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732514220019 2024-11-25T05:57:02,689 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a11d4a004434a2cb1c46cacde8c0f1a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732514221167 2024-11-25T05:57:02,697 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 39bebe481dd46e37e6dae001fcad1e53#C#compaction#388 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:02,698 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/66673ea4dfce450abc11133615797c21 is 50, key is test_row_0/C:col10/1732514221825/Put/seqid=0 2024-11-25T05:57:02,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742283_1459 (size=12104) 2024-11-25T05:57:02,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-25T05:57:02,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:02,714 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 39bebe481dd46e37e6dae001fcad1e53 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-25T05:57:02,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=A 2024-11-25T05:57:02,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:02,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=B 2024-11-25T05:57:02,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:02,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=C 2024-11-25T05:57:02,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:02,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/a528ba6761ea4819ab4d38404c73416a is 50, key is test_row_0/A:col10/1732514221889/Put/seqid=0 2024-11-25T05:57:02,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742284_1460 (size=12001) 2024-11-25T05:57:02,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-25T05:57:02,820 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T05:57:03,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on 39bebe481dd46e37e6dae001fcad1e53 2024-11-25T05:57:03,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:03,084 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/706de24bb95c4636be0d45705f19a869 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/706de24bb95c4636be0d45705f19a869 2024-11-25T05:57:03,087 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 39bebe481dd46e37e6dae001fcad1e53/B of 39bebe481dd46e37e6dae001fcad1e53 into 706de24bb95c4636be0d45705f19a869(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:03,087 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:03,087 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., storeName=39bebe481dd46e37e6dae001fcad1e53/B, priority=13, startTime=1732514222662; duration=0sec 2024-11-25T05:57:03,087 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:03,087 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 39bebe481dd46e37e6dae001fcad1e53:B 2024-11-25T05:57:03,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:03,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514283090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:03,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:03,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:03,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:03,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514283095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:03,105 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/66673ea4dfce450abc11133615797c21 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/66673ea4dfce450abc11133615797c21 2024-11-25T05:57:03,109 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 39bebe481dd46e37e6dae001fcad1e53/C of 39bebe481dd46e37e6dae001fcad1e53 into 66673ea4dfce450abc11133615797c21(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:03,109 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:03,109 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., storeName=39bebe481dd46e37e6dae001fcad1e53/C, priority=13, startTime=1732514222662; duration=0sec 2024-11-25T05:57:03,109 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:03,109 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 39bebe481dd46e37e6dae001fcad1e53:C 2024-11-25T05:57:03,123 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/a528ba6761ea4819ab4d38404c73416a 2024-11-25T05:57:03,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/72bee1f7a6e74beaba1d9dbee199ee76 is 50, key is test_row_0/B:col10/1732514221889/Put/seqid=0 2024-11-25T05:57:03,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742285_1461 (size=12001) 2024-11-25T05:57:03,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:03,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514283202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:03,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:03,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:03,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:03,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:03,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514283212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:03,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:03,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:03,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:03,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-25T05:57:03,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:03,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514283412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:03,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:03,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:03,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:03,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:03,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514283433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:03,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:03,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:03,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:03,534 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/72bee1f7a6e74beaba1d9dbee199ee76 2024-11-25T05:57:03,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/2530c9c7103341919f8ce34670947052 is 50, key is test_row_0/C:col10/1732514221889/Put/seqid=0 2024-11-25T05:57:03,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742286_1462 (size=12001) 2024-11-25T05:57:03,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:03,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514283722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:03,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:03,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:03,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:03,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:03,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514283742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:03,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:03,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:03,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:03,943 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/2530c9c7103341919f8ce34670947052 2024-11-25T05:57:03,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/a528ba6761ea4819ab4d38404c73416a as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/a528ba6761ea4819ab4d38404c73416a 2024-11-25T05:57:03,948 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/a528ba6761ea4819ab4d38404c73416a, entries=150, sequenceid=79, filesize=11.7 K 2024-11-25T05:57:03,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/72bee1f7a6e74beaba1d9dbee199ee76 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/72bee1f7a6e74beaba1d9dbee199ee76 2024-11-25T05:57:03,951 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/72bee1f7a6e74beaba1d9dbee199ee76, entries=150, sequenceid=79, filesize=11.7 K 2024-11-25T05:57:03,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/2530c9c7103341919f8ce34670947052 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/2530c9c7103341919f8ce34670947052 2024-11-25T05:57:03,955 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/2530c9c7103341919f8ce34670947052, entries=150, sequenceid=79, filesize=11.7 K 2024-11-25T05:57:03,955 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 39bebe481dd46e37e6dae001fcad1e53 in 1241ms, sequenceid=79, compaction requested=false 2024-11-25T05:57:03,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:03,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:03,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-25T05:57:03,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-25T05:57:03,958 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-25T05:57:03,958 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8520 sec 2024-11-25T05:57:03,959 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.8560 sec 2024-11-25T05:57:04,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on 39bebe481dd46e37e6dae001fcad1e53 2024-11-25T05:57:04,235 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 39bebe481dd46e37e6dae001fcad1e53 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-25T05:57:04,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=A 2024-11-25T05:57:04,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:04,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=B 2024-11-25T05:57:04,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:04,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=C 2024-11-25T05:57:04,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:04,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/7f003ab58d4646969f586fa5d8f97876 is 50, key is test_row_0/A:col10/1732514223094/Put/seqid=0 2024-11-25T05:57:04,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742287_1463 (size=14341) 2024-11-25T05:57:04,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-25T05:57:04,252 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:57:04,252 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:57:04,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-25T05:57:04,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-25T05:57:04,254 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:57:04,255 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:57:04,255 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:57:04,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:04,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514284262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:04,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:04,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:04,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:04,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:04,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34982 deadline: 1732514284267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:04,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:04,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35022 deadline: 1732514284268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:04,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:04,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:04,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:04,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:04,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:04,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:04,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:04,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34996 deadline: 1732514284270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:04,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:04,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514284274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:04,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:04,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:04,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:04,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:04,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:04,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:04,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-25T05:57:04,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:04,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34982 deadline: 1732514284382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:04,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:04,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:04,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:04,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:04,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35022 deadline: 1732514284382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:04,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:04,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:04,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:04,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:04,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34996 deadline: 1732514284391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:04,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:04,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:04,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:04,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:04,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514284392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:04,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:04,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:04,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:04,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-25T05:57:04,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:04,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:04,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:04,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:04,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:04,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:04,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-25T05:57:04,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:04,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:04,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:04,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:04,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:04,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:04,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-25T05:57:04,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:04,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34982 deadline: 1732514284602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:04,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:04,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:04,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35022 deadline: 1732514284602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:04,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:04,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:04,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:04,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:04,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:04,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34996 deadline: 1732514284603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:04,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:04,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514284603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:04,642 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/7f003ab58d4646969f586fa5d8f97876 2024-11-25T05:57:04,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/8f5a9f1ab3ce47a5aa5c1bfa62067d0b is 50, key is test_row_0/B:col10/1732514223094/Put/seqid=0 2024-11-25T05:57:04,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742288_1464 (size=12001) 2024-11-25T05:57:04,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-25T05:57:04,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:04,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:04,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:04,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:04,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:04,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:04,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-25T05:57:04,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:04,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:04,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:04,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:04,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:04,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:04,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-25T05:57:04,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:04,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:04,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34982 deadline: 1732514284912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:04,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35022 deadline: 1732514284912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:04,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:04,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:04,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:04,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:04,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514284912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:04,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:04,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:04,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:04,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34996 deadline: 1732514284912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:04,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:04,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:04,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:04,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:04,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:04,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:04,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:05,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-25T05:57:05,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:05,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:05,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:05,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:05,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:05,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:05,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/8f5a9f1ab3ce47a5aa5c1bfa62067d0b 2024-11-25T05:57:05,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/d48078767d324226b291b3d8bd4730bd is 50, key is test_row_0/C:col10/1732514223094/Put/seqid=0 2024-11-25T05:57:05,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742289_1465 (size=12001) 2024-11-25T05:57:05,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-25T05:57:05,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:05,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:05,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:05,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:05,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:05,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:05,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:05,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514285292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:05,298 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_0' in 39bebe481dd46e37e6dae001fcad1e53 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:57:05,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:05,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:05,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:05,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-25T05:57:05,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:05,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:05,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:05,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:05,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:05,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:05,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-25T05:57:05,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:05,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34982 deadline: 1732514285431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:05,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:05,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35022 deadline: 1732514285432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:05,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:05,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514285432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:05,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:05,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:05,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:05,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:05,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34996 deadline: 1732514285432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:05,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:05,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:05,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:05,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/d48078767d324226b291b3d8bd4730bd 2024-11-25T05:57:05,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/7f003ab58d4646969f586fa5d8f97876 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/7f003ab58d4646969f586fa5d8f97876 2024-11-25T05:57:05,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/7f003ab58d4646969f586fa5d8f97876, entries=200, sequenceid=94, filesize=14.0 K 2024-11-25T05:57:05,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/8f5a9f1ab3ce47a5aa5c1bfa62067d0b as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/8f5a9f1ab3ce47a5aa5c1bfa62067d0b 2024-11-25T05:57:05,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/8f5a9f1ab3ce47a5aa5c1bfa62067d0b, entries=150, sequenceid=94, filesize=11.7 K 2024-11-25T05:57:05,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/d48078767d324226b291b3d8bd4730bd as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/d48078767d324226b291b3d8bd4730bd 2024-11-25T05:57:05,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-25T05:57:05,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:05,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:05,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:05,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:05,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:05,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/d48078767d324226b291b3d8bd4730bd, entries=150, sequenceid=94, filesize=11.7 K 2024-11-25T05:57:05,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:05,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 39bebe481dd46e37e6dae001fcad1e53 in 1242ms, sequenceid=94, compaction requested=true 2024-11-25T05:57:05,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:05,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 39bebe481dd46e37e6dae001fcad1e53:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:57:05,477 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:57:05,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:05,477 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:57:05,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 39bebe481dd46e37e6dae001fcad1e53:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:57:05,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:05,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 39bebe481dd46e37e6dae001fcad1e53:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:57:05,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:57:05,478 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:57:05,478 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): 39bebe481dd46e37e6dae001fcad1e53/B is initiating minor compaction (all files) 2024-11-25T05:57:05,478 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 39bebe481dd46e37e6dae001fcad1e53/B in TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:05,478 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/706de24bb95c4636be0d45705f19a869, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/72bee1f7a6e74beaba1d9dbee199ee76, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/8f5a9f1ab3ce47a5aa5c1bfa62067d0b] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp, totalSize=35.3 K 2024-11-25T05:57:05,479 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:57:05,479 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): 39bebe481dd46e37e6dae001fcad1e53/A is initiating minor compaction (all files) 2024-11-25T05:57:05,479 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 39bebe481dd46e37e6dae001fcad1e53/A in TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:05,479 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/7e984871f62e4f86af09a902842582fa, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/a528ba6761ea4819ab4d38404c73416a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/7f003ab58d4646969f586fa5d8f97876] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp, totalSize=37.5 K 2024-11-25T05:57:05,479 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 706de24bb95c4636be0d45705f19a869, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732514221167 2024-11-25T05:57:05,479 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e984871f62e4f86af09a902842582fa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732514221167 2024-11-25T05:57:05,479 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 72bee1f7a6e74beaba1d9dbee199ee76, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732514221871 2024-11-25T05:57:05,479 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting a528ba6761ea4819ab4d38404c73416a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732514221871 2024-11-25T05:57:05,479 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f5a9f1ab3ce47a5aa5c1bfa62067d0b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732514223068 2024-11-25T05:57:05,480 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f003ab58d4646969f586fa5d8f97876, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732514223068 2024-11-25T05:57:05,485 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 39bebe481dd46e37e6dae001fcad1e53#A#compaction#395 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:05,485 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 39bebe481dd46e37e6dae001fcad1e53#B#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:05,486 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/acd48d58142e48b4a87e27b6e263706a is 50, key is test_row_0/A:col10/1732514223094/Put/seqid=0 2024-11-25T05:57:05,486 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/ce80ea2cc39a4b73baa5c0ccf1cdedf3 is 50, key is test_row_0/B:col10/1732514223094/Put/seqid=0 2024-11-25T05:57:05,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742291_1467 (size=12207) 2024-11-25T05:57:05,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742290_1466 (size=12207) 2024-11-25T05:57:05,496 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/ce80ea2cc39a4b73baa5c0ccf1cdedf3 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/ce80ea2cc39a4b73baa5c0ccf1cdedf3 2024-11-25T05:57:05,507 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 39bebe481dd46e37e6dae001fcad1e53/B of 39bebe481dd46e37e6dae001fcad1e53 into ce80ea2cc39a4b73baa5c0ccf1cdedf3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:05,508 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:05,508 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., storeName=39bebe481dd46e37e6dae001fcad1e53/B, priority=13, startTime=1732514225477; duration=0sec 2024-11-25T05:57:05,508 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:57:05,508 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 39bebe481dd46e37e6dae001fcad1e53:B 2024-11-25T05:57:05,508 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:57:05,508 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:57:05,508 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): 39bebe481dd46e37e6dae001fcad1e53/C is initiating minor compaction (all files) 2024-11-25T05:57:05,508 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 39bebe481dd46e37e6dae001fcad1e53/C in TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:05,509 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/66673ea4dfce450abc11133615797c21, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/2530c9c7103341919f8ce34670947052, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/d48078767d324226b291b3d8bd4730bd] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp, totalSize=35.3 K 2024-11-25T05:57:05,509 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 66673ea4dfce450abc11133615797c21, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732514221167 2024-11-25T05:57:05,509 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 2530c9c7103341919f8ce34670947052, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732514221871 2024-11-25T05:57:05,509 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting d48078767d324226b291b3d8bd4730bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732514223068 2024-11-25T05:57:05,514 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 39bebe481dd46e37e6dae001fcad1e53#C#compaction#397 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:05,515 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/6fdf7d58f7fa4a8c879a9d50ce10a68d is 50, key is test_row_0/C:col10/1732514223094/Put/seqid=0 2024-11-25T05:57:05,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742292_1468 (size=12207) 2024-11-25T05:57:05,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-25T05:57:05,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:05,628 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 39bebe481dd46e37e6dae001fcad1e53 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-25T05:57:05,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=A 2024-11-25T05:57:05,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:05,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=B 2024-11-25T05:57:05,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:05,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=C 2024-11-25T05:57:05,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:05,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/8ba4f1610dfd4a73998ad90598066da1 is 50, key is test_row_0/A:col10/1732514224268/Put/seqid=0 2024-11-25T05:57:05,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742293_1469 (size=12001) 2024-11-25T05:57:05,894 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/acd48d58142e48b4a87e27b6e263706a as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/acd48d58142e48b4a87e27b6e263706a 2024-11-25T05:57:05,898 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 39bebe481dd46e37e6dae001fcad1e53/A of 39bebe481dd46e37e6dae001fcad1e53 into acd48d58142e48b4a87e27b6e263706a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:05,898 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:05,898 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., storeName=39bebe481dd46e37e6dae001fcad1e53/A, priority=13, startTime=1732514225477; duration=0sec 2024-11-25T05:57:05,899 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:05,899 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 39bebe481dd46e37e6dae001fcad1e53:A 2024-11-25T05:57:05,921 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/6fdf7d58f7fa4a8c879a9d50ce10a68d as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/6fdf7d58f7fa4a8c879a9d50ce10a68d 2024-11-25T05:57:05,925 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 39bebe481dd46e37e6dae001fcad1e53/C of 39bebe481dd46e37e6dae001fcad1e53 into 6fdf7d58f7fa4a8c879a9d50ce10a68d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:05,925 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:05,925 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., storeName=39bebe481dd46e37e6dae001fcad1e53/C, priority=13, startTime=1732514225477; duration=0sec 2024-11-25T05:57:05,925 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:05,925 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 39bebe481dd46e37e6dae001fcad1e53:C 2024-11-25T05:57:06,037 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/8ba4f1610dfd4a73998ad90598066da1 2024-11-25T05:57:06,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/8e51ca1b06d54ce0b0026bf179980f4e is 50, key is test_row_0/B:col10/1732514224268/Put/seqid=0 2024-11-25T05:57:06,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742294_1470 (size=12001) 2024-11-25T05:57:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-25T05:57:06,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on 39bebe481dd46e37e6dae001fcad1e53 2024-11-25T05:57:06,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:06,446 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/8e51ca1b06d54ce0b0026bf179980f4e 2024-11-25T05:57:06,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/f4ba49f743dd4246829e42e38c10970f is 50, key is test_row_0/C:col10/1732514224268/Put/seqid=0 2024-11-25T05:57:06,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742295_1471 (size=12001) 2024-11-25T05:57:06,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:06,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34982 deadline: 1732514286459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:06,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:06,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:06,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:06,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34996 deadline: 1732514286460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:06,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:06,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:06,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:06,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35022 deadline: 1732514286461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:06,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:06,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:06,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:06,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514286464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:06,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:06,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:06,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:06,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:06,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34982 deadline: 1732514286572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:06,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:06,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:06,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:06,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34996 deadline: 1732514286582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:06,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35022 deadline: 1732514286582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:06,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514286582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:06,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:06,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:06,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:06,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34982 deadline: 1732514286781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:06,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:06,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:06,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:06,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:06,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34996 deadline: 1732514286792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:06,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:06,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:06,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:06,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:06,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35022 deadline: 1732514286792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:06,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:06,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:06,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:06,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:06,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514286792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:06,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:06,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:06,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:06,862 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/f4ba49f743dd4246829e42e38c10970f 2024-11-25T05:57:06,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/8ba4f1610dfd4a73998ad90598066da1 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/8ba4f1610dfd4a73998ad90598066da1 2024-11-25T05:57:06,867 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/8ba4f1610dfd4a73998ad90598066da1, entries=150, sequenceid=114, filesize=11.7 K 2024-11-25T05:57:06,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/8e51ca1b06d54ce0b0026bf179980f4e as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/8e51ca1b06d54ce0b0026bf179980f4e 2024-11-25T05:57:06,870 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/8e51ca1b06d54ce0b0026bf179980f4e, entries=150, sequenceid=114, filesize=11.7 K 2024-11-25T05:57:06,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/f4ba49f743dd4246829e42e38c10970f as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/f4ba49f743dd4246829e42e38c10970f 2024-11-25T05:57:06,873 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/f4ba49f743dd4246829e42e38c10970f, entries=150, sequenceid=114, filesize=11.7 K 2024-11-25T05:57:06,874 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 39bebe481dd46e37e6dae001fcad1e53 in 1246ms, sequenceid=114, compaction requested=false 2024-11-25T05:57:06,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:06,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:06,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-25T05:57:06,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-25T05:57:06,877 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-25T05:57:06,877 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6200 sec 2024-11-25T05:57:06,879 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.6250 sec 2024-11-25T05:57:07,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on 39bebe481dd46e37e6dae001fcad1e53 2024-11-25T05:57:07,096 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 39bebe481dd46e37e6dae001fcad1e53 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-25T05:57:07,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=A 2024-11-25T05:57:07,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:07,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=B 2024-11-25T05:57:07,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:07,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=C 2024-11-25T05:57:07,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:07,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/44e3e7c4f2d44323a073c80906ec7225 is 50, key is test_row_0/A:col10/1732514226459/Put/seqid=0 2024-11-25T05:57:07,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742296_1472 (size=14391) 2024-11-25T05:57:07,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35022 deadline: 1732514287123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:07,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:07,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:07,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:07,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34982 deadline: 1732514287126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:07,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:07,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:07,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:07,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34996 deadline: 1732514287128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:07,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:07,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:07,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:07,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:07,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514287133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:07,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:07,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:07,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:07,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:07,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35022 deadline: 1732514287242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:07,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:07,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:07,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34982 deadline: 1732514287242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:07,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:07,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:07,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:07,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34996 deadline: 1732514287242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:07,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:07,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:07,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:07,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514287242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:07,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:07,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:07,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:07,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:07,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:07,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:07,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514287322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:07,328 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_0' in 39bebe481dd46e37e6dae001fcad1e53 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:57:07,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:07,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:07,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:07,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:07,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35022 deadline: 1732514287452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:07,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:07,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:07,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:07,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:07,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34982 deadline: 1732514287452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:07,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:07,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:07,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:07,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:07,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34996 deadline: 1732514287452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:07,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:07,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:07,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:07,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:07,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514287452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:07,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:07,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:07,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:07,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/44e3e7c4f2d44323a073c80906ec7225 2024-11-25T05:57:07,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/4a87c71333284cc08829b47a4b930ae8 is 50, key is test_row_0/B:col10/1732514226459/Put/seqid=0 2024-11-25T05:57:07,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742297_1473 (size=12051) 2024-11-25T05:57:07,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:07,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34996 deadline: 1732514287772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:07,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:07,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:07,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:07,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:07,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514287772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:07,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:07,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:07,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:07,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:07,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35022 deadline: 1732514287772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:07,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:07,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:07,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:07,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:07,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34982 deadline: 1732514287772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:07,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:07,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:07,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:07,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/4a87c71333284cc08829b47a4b930ae8 2024-11-25T05:57:07,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/eea796ba5d0a4cd8b0c3a2d7e87ee883 is 50, key is test_row_0/C:col10/1732514226459/Put/seqid=0 2024-11-25T05:57:07,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742298_1474 (size=12051) 2024-11-25T05:57:08,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:08,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34996 deadline: 1732514288282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:08,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:08,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:08,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:08,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:08,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514288282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:08,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:08,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:08,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:08,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:08,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35022 deadline: 1732514288282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:08,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:08,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:08,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:08,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:08,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34982 deadline: 1732514288292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:08,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:08,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:08,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:08,324 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/eea796ba5d0a4cd8b0c3a2d7e87ee883 2024-11-25T05:57:08,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/44e3e7c4f2d44323a073c80906ec7225 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/44e3e7c4f2d44323a073c80906ec7225 2024-11-25T05:57:08,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/44e3e7c4f2d44323a073c80906ec7225, entries=200, sequenceid=130, filesize=14.1 K 2024-11-25T05:57:08,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/4a87c71333284cc08829b47a4b930ae8 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/4a87c71333284cc08829b47a4b930ae8 2024-11-25T05:57:08,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/4a87c71333284cc08829b47a4b930ae8, entries=150, sequenceid=130, filesize=11.8 K 2024-11-25T05:57:08,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/eea796ba5d0a4cd8b0c3a2d7e87ee883 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/eea796ba5d0a4cd8b0c3a2d7e87ee883 2024-11-25T05:57:08,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/eea796ba5d0a4cd8b0c3a2d7e87ee883, entries=150, sequenceid=130, filesize=11.8 K 2024-11-25T05:57:08,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 39bebe481dd46e37e6dae001fcad1e53 in 1242ms, sequenceid=130, compaction requested=true 2024-11-25T05:57:08,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:08,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 39bebe481dd46e37e6dae001fcad1e53:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:57:08,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:08,338 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:57:08,338 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:57:08,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 39bebe481dd46e37e6dae001fcad1e53:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:57:08,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:08,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 39bebe481dd46e37e6dae001fcad1e53:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:57:08,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:57:08,339 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38599 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:57:08,339 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): 39bebe481dd46e37e6dae001fcad1e53/A is initiating minor compaction (all files) 2024-11-25T05:57:08,339 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 39bebe481dd46e37e6dae001fcad1e53/A in TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:08,339 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/acd48d58142e48b4a87e27b6e263706a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/8ba4f1610dfd4a73998ad90598066da1, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/44e3e7c4f2d44323a073c80906ec7225] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp, totalSize=37.7 K 2024-11-25T05:57:08,339 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:57:08,339 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): 39bebe481dd46e37e6dae001fcad1e53/B is initiating minor compaction (all files) 2024-11-25T05:57:08,339 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting acd48d58142e48b4a87e27b6e263706a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732514223068 2024-11-25T05:57:08,339 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 39bebe481dd46e37e6dae001fcad1e53/B in TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:08,339 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/ce80ea2cc39a4b73baa5c0ccf1cdedf3, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/8e51ca1b06d54ce0b0026bf179980f4e, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/4a87c71333284cc08829b47a4b930ae8] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp, totalSize=35.4 K 2024-11-25T05:57:08,340 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ba4f1610dfd4a73998ad90598066da1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732514224267 2024-11-25T05:57:08,340 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce80ea2cc39a4b73baa5c0ccf1cdedf3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732514223068 2024-11-25T05:57:08,340 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 44e3e7c4f2d44323a073c80906ec7225, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732514226458 2024-11-25T05:57:08,340 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e51ca1b06d54ce0b0026bf179980f4e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732514224267 2024-11-25T05:57:08,340 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a87c71333284cc08829b47a4b930ae8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732514226458 2024-11-25T05:57:08,346 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 39bebe481dd46e37e6dae001fcad1e53#B#compaction#404 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:08,346 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 39bebe481dd46e37e6dae001fcad1e53#A#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:08,346 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/b7fa76a0f7004eee8977da15bb98ecfb is 50, key is test_row_0/B:col10/1732514226459/Put/seqid=0 2024-11-25T05:57:08,346 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/21298ef611ef470a804c3173fdf49a6f is 50, key is test_row_0/A:col10/1732514226459/Put/seqid=0 2024-11-25T05:57:08,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742299_1475 (size=12359) 2024-11-25T05:57:08,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742300_1476 (size=12359) 2024-11-25T05:57:08,353 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/b7fa76a0f7004eee8977da15bb98ecfb as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/b7fa76a0f7004eee8977da15bb98ecfb 2024-11-25T05:57:08,356 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 39bebe481dd46e37e6dae001fcad1e53/B of 39bebe481dd46e37e6dae001fcad1e53 into b7fa76a0f7004eee8977da15bb98ecfb(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:08,356 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:08,356 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., storeName=39bebe481dd46e37e6dae001fcad1e53/B, priority=13, startTime=1732514228338; duration=0sec 2024-11-25T05:57:08,356 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:57:08,356 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 39bebe481dd46e37e6dae001fcad1e53:B 2024-11-25T05:57:08,356 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:57:08,356 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:57:08,357 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): 39bebe481dd46e37e6dae001fcad1e53/C is initiating minor compaction (all files) 2024-11-25T05:57:08,357 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 39bebe481dd46e37e6dae001fcad1e53/C in TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:08,357 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/6fdf7d58f7fa4a8c879a9d50ce10a68d, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/f4ba49f743dd4246829e42e38c10970f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/eea796ba5d0a4cd8b0c3a2d7e87ee883] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp, totalSize=35.4 K 2024-11-25T05:57:08,357 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fdf7d58f7fa4a8c879a9d50ce10a68d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732514223068 2024-11-25T05:57:08,357 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4ba49f743dd4246829e42e38c10970f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732514224267 2024-11-25T05:57:08,358 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting eea796ba5d0a4cd8b0c3a2d7e87ee883, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732514226458 2024-11-25T05:57:08,363 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 39bebe481dd46e37e6dae001fcad1e53#C#compaction#406 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:08,363 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/066bc24caf2b4d5980f20c3227be0b97 is 50, key is test_row_0/C:col10/1732514226459/Put/seqid=0 2024-11-25T05:57:08,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742301_1477 (size=12359) 2024-11-25T05:57:08,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-25T05:57:08,412 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:57:08,413 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:57:08,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-25T05:57:08,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-25T05:57:08,414 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:57:08,415 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:57:08,415 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:57:08,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-25T05:57:08,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-25T05:57:08,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:08,567 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 39bebe481dd46e37e6dae001fcad1e53 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-25T05:57:08,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=A 2024-11-25T05:57:08,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:08,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=B 2024-11-25T05:57:08,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:08,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=C 2024-11-25T05:57:08,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:08,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/1150a10764d54af3a02df9f6401cada4 is 50, key is test_row_0/A:col10/1732514227126/Put/seqid=0 2024-11-25T05:57:08,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742302_1478 (size=12151) 2024-11-25T05:57:08,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-25T05:57:08,754 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/21298ef611ef470a804c3173fdf49a6f as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/21298ef611ef470a804c3173fdf49a6f 2024-11-25T05:57:08,757 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 39bebe481dd46e37e6dae001fcad1e53/A of 39bebe481dd46e37e6dae001fcad1e53 into 21298ef611ef470a804c3173fdf49a6f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:08,757 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:08,757 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., storeName=39bebe481dd46e37e6dae001fcad1e53/A, priority=13, startTime=1732514228338; duration=0sec 2024-11-25T05:57:08,757 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:08,757 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 39bebe481dd46e37e6dae001fcad1e53:A 2024-11-25T05:57:08,764 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-25T05:57:08,764 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-25T05:57:08,770 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/066bc24caf2b4d5980f20c3227be0b97 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/066bc24caf2b4d5980f20c3227be0b97 2024-11-25T05:57:08,774 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 39bebe481dd46e37e6dae001fcad1e53/C of 39bebe481dd46e37e6dae001fcad1e53 into 066bc24caf2b4d5980f20c3227be0b97(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:08,774 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:08,774 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., storeName=39bebe481dd46e37e6dae001fcad1e53/C, priority=13, startTime=1732514228338; duration=0sec 2024-11-25T05:57:08,774 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:08,774 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 39bebe481dd46e37e6dae001fcad1e53:C 2024-11-25T05:57:08,974 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/1150a10764d54af3a02df9f6401cada4 2024-11-25T05:57:08,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/216bb1892c3648f0895c8b237909ba58 is 50, key is test_row_0/B:col10/1732514227126/Put/seqid=0 2024-11-25T05:57:08,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742303_1479 (size=12151) 2024-11-25T05:57:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-25T05:57:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on 39bebe481dd46e37e6dae001fcad1e53 2024-11-25T05:57:09,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:09,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35022 deadline: 1732514289321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:09,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:09,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:09,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:09,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34996 deadline: 1732514289323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:09,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:09,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:09,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514289323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:09,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:09,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:09,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34982 deadline: 1732514289323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:09,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:09,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:09,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:09,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:09,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:09,386 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/216bb1892c3648f0895c8b237909ba58 2024-11-25T05:57:09,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/100b45950d144ed9b004bcf1fed2f5a3 is 50, key is test_row_0/C:col10/1732514227126/Put/seqid=0 2024-11-25T05:57:09,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742304_1480 (size=12151) 2024-11-25T05:57:09,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35022 deadline: 1732514289432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:09,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:09,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:09,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:09,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:09,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34996 deadline: 1732514289442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:09,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:09,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:09,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:09,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514289442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:09,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:09,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:09,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:09,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:09,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:09,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34982 deadline: 1732514289442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:09,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:09,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:09,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:09,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-25T05:57:09,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:09,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35022 deadline: 1732514289641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:09,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:09,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:09,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:09,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:09,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34996 deadline: 1732514289652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:09,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:09,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:09,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:09,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514289652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:09,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:09,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:09,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:09,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:09,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:09,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34982 deadline: 1732514289652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:09,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:09,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:09,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:09,799 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/100b45950d144ed9b004bcf1fed2f5a3 2024-11-25T05:57:09,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/1150a10764d54af3a02df9f6401cada4 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/1150a10764d54af3a02df9f6401cada4 2024-11-25T05:57:09,805 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/1150a10764d54af3a02df9f6401cada4, entries=150, sequenceid=149, filesize=11.9 K 2024-11-25T05:57:09,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/216bb1892c3648f0895c8b237909ba58 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/216bb1892c3648f0895c8b237909ba58 2024-11-25T05:57:09,808 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/216bb1892c3648f0895c8b237909ba58, entries=150, sequenceid=149, filesize=11.9 K 2024-11-25T05:57:09,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/100b45950d144ed9b004bcf1fed2f5a3 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/100b45950d144ed9b004bcf1fed2f5a3 2024-11-25T05:57:09,811 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/100b45950d144ed9b004bcf1fed2f5a3, entries=150, sequenceid=149, filesize=11.9 K 2024-11-25T05:57:09,811 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 39bebe481dd46e37e6dae001fcad1e53 in 1244ms, sequenceid=149, compaction requested=false 2024-11-25T05:57:09,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:09,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:09,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-25T05:57:09,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-25T05:57:09,814 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-25T05:57:09,814 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3980 sec 2024-11-25T05:57:09,816 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.4020 sec 2024-11-25T05:57:09,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on 39bebe481dd46e37e6dae001fcad1e53 2024-11-25T05:57:09,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 39bebe481dd46e37e6dae001fcad1e53 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-25T05:57:09,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=A 2024-11-25T05:57:09,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:09,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=B 2024-11-25T05:57:09,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:09,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=C 2024-11-25T05:57:09,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:09,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/6744ee50790849e390c927a73d295d72 is 50, key is test_row_0/A:col10/1732514229311/Put/seqid=0 2024-11-25T05:57:09,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742305_1481 (size=14541) 2024-11-25T05:57:09,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:09,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34982 deadline: 1732514289982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:09,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:09,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514289984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:09,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:09,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:09,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:09,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:09,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:09,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:09,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34996 deadline: 1732514289997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:09,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:09,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:09,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:09,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:10,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:10,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35022 deadline: 1732514289998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:10,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:10,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:10,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:10,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:10,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34982 deadline: 1732514290102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:10,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:10,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:10,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514290102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:10,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:10,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:10,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:10,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34996 deadline: 1732514290102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:10,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:10,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35022 deadline: 1732514290122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:10,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:10,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:10,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:10,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:10,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34982 deadline: 1732514290312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:10,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:10,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:10,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:10,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514290312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:10,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:10,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:10,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:10,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:10,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:10,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34996 deadline: 1732514290312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:10,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:10,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:10,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:10,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:10,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35022 deadline: 1732514290342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:10,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:10,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:10,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:10,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/6744ee50790849e390c927a73d295d72 2024-11-25T05:57:10,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/1ded07da516b40ffb0561e68eacd8fe8 is 50, key is test_row_0/B:col10/1732514229311/Put/seqid=0 2024-11-25T05:57:10,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742306_1482 (size=12151) 2024-11-25T05:57:10,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-25T05:57:10,562 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:57:10,563 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:57:10,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-25T05:57:10,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T05:57:10,564 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:57:10,565 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:57:10,565 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:57:10,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:10,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34982 deadline: 1732514290621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:10,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:10,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:10,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514290622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:10,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:10,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:10,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:10,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:10,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:10,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:10,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34996 deadline: 1732514290622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:10,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:10,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:10,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:10,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:10,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35022 deadline: 1732514290662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:10,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:10,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:10,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:10,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T05:57:10,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T05:57:10,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:10,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:10,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:10,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:10,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:10,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:10,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/1ded07da516b40ffb0561e68eacd8fe8 2024-11-25T05:57:10,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/07876dfe376449859c1df6b6e9c39085 is 50, key is test_row_0/C:col10/1732514229311/Put/seqid=0 2024-11-25T05:57:10,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742307_1483 (size=12151) 2024-11-25T05:57:10,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T05:57:10,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:10,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:10,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:10,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:10,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:10,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:10,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T05:57:11,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T05:57:11,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:11,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:11,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:11,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:11,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:11,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:11,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:11,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34982 deadline: 1732514291132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:11,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:11,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:11,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:11,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:11,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514291142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:11,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:11,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:11,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34996 deadline: 1732514291142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:11,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:11,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:11,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:11,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:11,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:11,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T05:57:11,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:11,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:11,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:11,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:11,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:11,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:11,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:11,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35022 deadline: 1732514291171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:11,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:11,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:11,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:11,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/07876dfe376449859c1df6b6e9c39085 2024-11-25T05:57:11,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/6744ee50790849e390c927a73d295d72 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/6744ee50790849e390c927a73d295d72 2024-11-25T05:57:11,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/6744ee50790849e390c927a73d295d72, entries=200, sequenceid=166, filesize=14.2 K 2024-11-25T05:57:11,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/1ded07da516b40ffb0561e68eacd8fe8 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/1ded07da516b40ffb0561e68eacd8fe8 2024-11-25T05:57:11,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T05:57:11,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/1ded07da516b40ffb0561e68eacd8fe8, entries=150, sequenceid=166, filesize=11.9 K 2024-11-25T05:57:11,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/07876dfe376449859c1df6b6e9c39085 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/07876dfe376449859c1df6b6e9c39085 2024-11-25T05:57:11,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/07876dfe376449859c1df6b6e9c39085, entries=150, sequenceid=166, filesize=11.9 K 2024-11-25T05:57:11,198 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 39bebe481dd46e37e6dae001fcad1e53 in 1242ms, sequenceid=166, compaction requested=true 2024-11-25T05:57:11,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:11,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 39bebe481dd46e37e6dae001fcad1e53:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:57:11,198 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:57:11,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:11,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 39bebe481dd46e37e6dae001fcad1e53:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:57:11,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:11,198 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:57:11,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 39bebe481dd46e37e6dae001fcad1e53:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:57:11,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:57:11,199 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39051 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:57:11,199 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:57:11,199 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): 39bebe481dd46e37e6dae001fcad1e53/A is initiating minor compaction (all files) 2024-11-25T05:57:11,199 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): 39bebe481dd46e37e6dae001fcad1e53/B is initiating minor compaction (all files) 2024-11-25T05:57:11,199 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 39bebe481dd46e37e6dae001fcad1e53/B in TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:11,199 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 39bebe481dd46e37e6dae001fcad1e53/A in TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:11,199 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/b7fa76a0f7004eee8977da15bb98ecfb, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/216bb1892c3648f0895c8b237909ba58, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/1ded07da516b40ffb0561e68eacd8fe8] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp, totalSize=35.8 K 2024-11-25T05:57:11,199 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/21298ef611ef470a804c3173fdf49a6f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/1150a10764d54af3a02df9f6401cada4, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/6744ee50790849e390c927a73d295d72] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp, totalSize=38.1 K 2024-11-25T05:57:11,199 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21298ef611ef470a804c3173fdf49a6f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732514226458 2024-11-25T05:57:11,199 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting b7fa76a0f7004eee8977da15bb98ecfb, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732514226458 2024-11-25T05:57:11,199 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 216bb1892c3648f0895c8b237909ba58, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732514227121 2024-11-25T05:57:11,199 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1150a10764d54af3a02df9f6401cada4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732514227121 2024-11-25T05:57:11,200 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ded07da516b40ffb0561e68eacd8fe8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732514229308 2024-11-25T05:57:11,200 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6744ee50790849e390c927a73d295d72, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732514229308 2024-11-25T05:57:11,205 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 39bebe481dd46e37e6dae001fcad1e53#B#compaction#413 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:11,205 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 39bebe481dd46e37e6dae001fcad1e53#A#compaction#414 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:11,206 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/5f15fbabb64f4298acbaf13ac9a36f89 is 50, key is test_row_0/B:col10/1732514229311/Put/seqid=0 2024-11-25T05:57:11,206 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/4d0e685c0e2c4bb3bc2bffff7bdf5597 is 50, key is test_row_0/A:col10/1732514229311/Put/seqid=0 2024-11-25T05:57:11,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742309_1485 (size=12561) 2024-11-25T05:57:11,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742308_1484 (size=12561) 2024-11-25T05:57:11,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T05:57:11,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:11,327 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 39bebe481dd46e37e6dae001fcad1e53 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-25T05:57:11,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=A 2024-11-25T05:57:11,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:11,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=B 2024-11-25T05:57:11,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:11,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=C 2024-11-25T05:57:11,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:11,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/48dd514cbf1b468397e28d93fcabf2a9 is 50, key is test_row_0/A:col10/1732514229996/Put/seqid=0 2024-11-25T05:57:11,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742310_1486 (size=12151) 2024-11-25T05:57:11,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on 39bebe481dd46e37e6dae001fcad1e53 2024-11-25T05:57:11,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:11,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:11,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514291450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:11,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:11,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:11,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:11,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:11,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514291562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:11,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:11,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:11,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:11,613 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/5f15fbabb64f4298acbaf13ac9a36f89 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/5f15fbabb64f4298acbaf13ac9a36f89 2024-11-25T05:57:11,613 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/4d0e685c0e2c4bb3bc2bffff7bdf5597 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/4d0e685c0e2c4bb3bc2bffff7bdf5597 2024-11-25T05:57:11,617 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 39bebe481dd46e37e6dae001fcad1e53/B of 39bebe481dd46e37e6dae001fcad1e53 into 5f15fbabb64f4298acbaf13ac9a36f89(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:11,617 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:11,617 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., storeName=39bebe481dd46e37e6dae001fcad1e53/B, priority=13, startTime=1732514231198; duration=0sec 2024-11-25T05:57:11,617 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:57:11,617 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 39bebe481dd46e37e6dae001fcad1e53:B 2024-11-25T05:57:11,617 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:57:11,618 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 39bebe481dd46e37e6dae001fcad1e53/A of 39bebe481dd46e37e6dae001fcad1e53 into 4d0e685c0e2c4bb3bc2bffff7bdf5597(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:11,618 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:11,618 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., storeName=39bebe481dd46e37e6dae001fcad1e53/A, priority=13, startTime=1732514231198; duration=0sec 2024-11-25T05:57:11,618 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:57:11,618 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): 39bebe481dd46e37e6dae001fcad1e53/C is initiating minor compaction (all files) 2024-11-25T05:57:11,618 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:11,618 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 39bebe481dd46e37e6dae001fcad1e53:A 2024-11-25T05:57:11,618 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 39bebe481dd46e37e6dae001fcad1e53/C in TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:11,618 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/066bc24caf2b4d5980f20c3227be0b97, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/100b45950d144ed9b004bcf1fed2f5a3, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/07876dfe376449859c1df6b6e9c39085] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp, totalSize=35.8 K 2024-11-25T05:57:11,618 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 066bc24caf2b4d5980f20c3227be0b97, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732514226458 2024-11-25T05:57:11,619 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 100b45950d144ed9b004bcf1fed2f5a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732514227121 2024-11-25T05:57:11,619 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 07876dfe376449859c1df6b6e9c39085, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732514229308 2024-11-25T05:57:11,623 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 39bebe481dd46e37e6dae001fcad1e53#C#compaction#416 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:11,624 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/34e6b5fb83934bc29ffa758c80a63fbe is 50, key is test_row_0/C:col10/1732514229311/Put/seqid=0 2024-11-25T05:57:11,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742311_1487 (size=12561) 2024-11-25T05:57:11,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T05:57:11,736 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/48dd514cbf1b468397e28d93fcabf2a9 2024-11-25T05:57:11,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/32567a00419841acad70aaa1edfaa17e is 50, key is test_row_0/B:col10/1732514229996/Put/seqid=0 2024-11-25T05:57:11,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742312_1488 (size=12151) 2024-11-25T05:57:11,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:11,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514291772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:12,030 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/34e6b5fb83934bc29ffa758c80a63fbe as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/34e6b5fb83934bc29ffa758c80a63fbe 2024-11-25T05:57:12,034 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 39bebe481dd46e37e6dae001fcad1e53/C of 39bebe481dd46e37e6dae001fcad1e53 into 34e6b5fb83934bc29ffa758c80a63fbe(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:12,034 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:12,034 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., storeName=39bebe481dd46e37e6dae001fcad1e53/C, priority=13, startTime=1732514231198; duration=0sec 2024-11-25T05:57:12,034 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:12,034 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 39bebe481dd46e37e6dae001fcad1e53:C 2024-11-25T05:57:12,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:12,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514292081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:12,147 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/32567a00419841acad70aaa1edfaa17e 2024-11-25T05:57:12,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/e40a034e04714cae88c0cb965c552b5e is 50, key is test_row_0/C:col10/1732514229996/Put/seqid=0 2024-11-25T05:57:12,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742313_1489 (size=12151) 2024-11-25T05:57:12,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:12,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34982 deadline: 1732514292152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:12,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:12,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514292152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:12,159 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_0' in 39bebe481dd46e37e6dae001fcad1e53 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:57:12,159 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_1' in 39bebe481dd46e37e6dae001fcad1e53 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:57:12,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:12,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:12,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:12,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:12,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:12,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:12,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:12,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34996 deadline: 1732514292161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:12,168 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_2' in 39bebe481dd46e37e6dae001fcad1e53 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:57:12,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:12,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:12,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:12,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35022 deadline: 1732514292192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:12,195 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_1' in 39bebe481dd46e37e6dae001fcad1e53 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:57:12,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:12,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:12,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:12,557 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/e40a034e04714cae88c0cb965c552b5e 2024-11-25T05:57:12,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/48dd514cbf1b468397e28d93fcabf2a9 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/48dd514cbf1b468397e28d93fcabf2a9 2024-11-25T05:57:12,563 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/48dd514cbf1b468397e28d93fcabf2a9, entries=150, sequenceid=183, filesize=11.9 K 2024-11-25T05:57:12,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/32567a00419841acad70aaa1edfaa17e as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/32567a00419841acad70aaa1edfaa17e 2024-11-25T05:57:12,566 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/32567a00419841acad70aaa1edfaa17e, entries=150, sequenceid=183, filesize=11.9 K 2024-11-25T05:57:12,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/e40a034e04714cae88c0cb965c552b5e as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/e40a034e04714cae88c0cb965c552b5e 2024-11-25T05:57:12,569 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/e40a034e04714cae88c0cb965c552b5e, entries=150, sequenceid=183, filesize=11.9 K 2024-11-25T05:57:12,570 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 39bebe481dd46e37e6dae001fcad1e53 in 1243ms, sequenceid=183, compaction requested=false 2024-11-25T05:57:12,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:12,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:12,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-25T05:57:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-25T05:57:12,572 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-25T05:57:12,572 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0060 sec 2024-11-25T05:57:12,574 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.0100 sec 2024-11-25T05:57:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on 39bebe481dd46e37e6dae001fcad1e53 2024-11-25T05:57:12,599 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 39bebe481dd46e37e6dae001fcad1e53 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-25T05:57:12,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=A 2024-11-25T05:57:12,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:12,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=B 2024-11-25T05:57:12,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:12,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=C 2024-11-25T05:57:12,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:12,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/b55a31378a6a42bea5c9b083b7f43bdf is 50, key is test_row_0/A:col10/1732514231448/Put/seqid=0 2024-11-25T05:57:12,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742314_1490 (size=14541) 2024-11-25T05:57:12,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:12,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514292683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:12,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:12,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:12,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:12,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T05:57:12,712 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:57:12,713 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:57:12,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-25T05:57:12,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T05:57:12,714 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:57:12,715 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:57:12,715 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:57:12,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:12,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514292792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:12,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T05:57:12,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T05:57:12,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:12,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:12,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:12,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:12,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:12,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:13,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/b55a31378a6a42bea5c9b083b7f43bdf 2024-11-25T05:57:13,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/8cff78c375a848af8cb423da66f1e630 is 50, key is test_row_0/B:col10/1732514231448/Put/seqid=0 2024-11-25T05:57:13,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742315_1491 (size=12151) 2024-11-25T05:57:13,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T05:57:13,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:13,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:13,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:13,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:13,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:13,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:13,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:13,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514293012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:13,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:13,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:13,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:13,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T05:57:13,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T05:57:13,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:13,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:13,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:13,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:13,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:13,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:13,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T05:57:13,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:13,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:13,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:13,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:13,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:13,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:13,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:13,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514293332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:13,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:13,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:13,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:13,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T05:57:13,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/8cff78c375a848af8cb423da66f1e630 2024-11-25T05:57:13,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/68682dddab7d4123bc8388281e3a4f05 is 50, key is test_row_0/C:col10/1732514231448/Put/seqid=0 2024-11-25T05:57:13,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742316_1492 (size=12151) 2024-11-25T05:57:13,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T05:57:13,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:13,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:13,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:13,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:13,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:13,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:13,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T05:57:13,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:13,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:13,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:13,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:13,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:13,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:13,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T05:57:13,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:13,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:13,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:13,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:13,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:13,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:13,825 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/68682dddab7d4123bc8388281e3a4f05 2024-11-25T05:57:13,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/b55a31378a6a42bea5c9b083b7f43bdf as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/b55a31378a6a42bea5c9b083b7f43bdf 2024-11-25T05:57:13,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/b55a31378a6a42bea5c9b083b7f43bdf, entries=200, sequenceid=202, filesize=14.2 K 2024-11-25T05:57:13,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/8cff78c375a848af8cb423da66f1e630 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/8cff78c375a848af8cb423da66f1e630 2024-11-25T05:57:13,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/8cff78c375a848af8cb423da66f1e630, entries=150, sequenceid=202, filesize=11.9 K 2024-11-25T05:57:13,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/68682dddab7d4123bc8388281e3a4f05 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/68682dddab7d4123bc8388281e3a4f05 2024-11-25T05:57:13,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/68682dddab7d4123bc8388281e3a4f05, entries=150, sequenceid=202, filesize=11.9 K 2024-11-25T05:57:13,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 39bebe481dd46e37e6dae001fcad1e53 in 1240ms, sequenceid=202, compaction requested=true 2024-11-25T05:57:13,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:13,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 39bebe481dd46e37e6dae001fcad1e53:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:57:13,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:13,840 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:57:13,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 39bebe481dd46e37e6dae001fcad1e53:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:57:13,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:13,840 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:57:13,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 39bebe481dd46e37e6dae001fcad1e53:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:57:13,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:57:13,840 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:57:13,840 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:57:13,840 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): 39bebe481dd46e37e6dae001fcad1e53/B is initiating minor compaction (all files) 2024-11-25T05:57:13,840 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): 39bebe481dd46e37e6dae001fcad1e53/A is initiating minor compaction (all files) 2024-11-25T05:57:13,841 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 39bebe481dd46e37e6dae001fcad1e53/A in TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:13,841 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 39bebe481dd46e37e6dae001fcad1e53/B in TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:13,841 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/5f15fbabb64f4298acbaf13ac9a36f89, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/32567a00419841acad70aaa1edfaa17e, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/8cff78c375a848af8cb423da66f1e630] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp, totalSize=36.0 K 2024-11-25T05:57:13,841 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/4d0e685c0e2c4bb3bc2bffff7bdf5597, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/48dd514cbf1b468397e28d93fcabf2a9, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/b55a31378a6a42bea5c9b083b7f43bdf] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp, totalSize=38.3 K 2024-11-25T05:57:13,841 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f15fbabb64f4298acbaf13ac9a36f89, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732514229308 2024-11-25T05:57:13,841 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d0e685c0e2c4bb3bc2bffff7bdf5597, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732514229308 2024-11-25T05:57:13,841 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48dd514cbf1b468397e28d93fcabf2a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732514229982 2024-11-25T05:57:13,841 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 32567a00419841acad70aaa1edfaa17e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732514229982 2024-11-25T05:57:13,841 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cff78c375a848af8cb423da66f1e630, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732514231416 2024-11-25T05:57:13,841 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting b55a31378a6a42bea5c9b083b7f43bdf, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732514231416 2024-11-25T05:57:13,846 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 39bebe481dd46e37e6dae001fcad1e53#A#compaction#422 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:13,846 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/4b8886dcc45a4fcf891f3e184664e1ee is 50, key is test_row_0/A:col10/1732514231448/Put/seqid=0 2024-11-25T05:57:13,848 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 39bebe481dd46e37e6dae001fcad1e53#B#compaction#423 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:13,849 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/1ba123900f4c429ba4299e19a0eabc2c is 50, key is test_row_0/B:col10/1732514231448/Put/seqid=0 2024-11-25T05:57:13,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T05:57:13,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742317_1493 (size=12663) 2024-11-25T05:57:13,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on 39bebe481dd46e37e6dae001fcad1e53 2024-11-25T05:57:13,856 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 39bebe481dd46e37e6dae001fcad1e53 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-25T05:57:13,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=A 2024-11-25T05:57:13,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:13,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=B 2024-11-25T05:57:13,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:13,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=C 2024-11-25T05:57:13,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:13,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742318_1494 (size=12663) 2024-11-25T05:57:13,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/8a8f5d6a29c04c2394b2065f42ccea4a is 50, key is test_row_0/A:col10/1732514232681/Put/seqid=0 2024-11-25T05:57:13,871 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/1ba123900f4c429ba4299e19a0eabc2c as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/1ba123900f4c429ba4299e19a0eabc2c 2024-11-25T05:57:13,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742319_1495 (size=14541) 2024-11-25T05:57:13,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/8a8f5d6a29c04c2394b2065f42ccea4a 2024-11-25T05:57:13,876 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 39bebe481dd46e37e6dae001fcad1e53/B of 39bebe481dd46e37e6dae001fcad1e53 into 1ba123900f4c429ba4299e19a0eabc2c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:13,877 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:13,877 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., storeName=39bebe481dd46e37e6dae001fcad1e53/B, priority=13, startTime=1732514233840; duration=0sec 2024-11-25T05:57:13,877 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:57:13,877 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 39bebe481dd46e37e6dae001fcad1e53:B 2024-11-25T05:57:13,877 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:57:13,877 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:57:13,877 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): 39bebe481dd46e37e6dae001fcad1e53/C is initiating minor compaction (all files) 2024-11-25T05:57:13,877 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 39bebe481dd46e37e6dae001fcad1e53/C in TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:13,877 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/34e6b5fb83934bc29ffa758c80a63fbe, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/e40a034e04714cae88c0cb965c552b5e, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/68682dddab7d4123bc8388281e3a4f05] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp, totalSize=36.0 K 2024-11-25T05:57:13,878 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 34e6b5fb83934bc29ffa758c80a63fbe, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732514229308 2024-11-25T05:57:13,878 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting e40a034e04714cae88c0cb965c552b5e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732514229982 2024-11-25T05:57:13,878 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 68682dddab7d4123bc8388281e3a4f05, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732514231416 2024-11-25T05:57:13,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/cca0c8c7c8de48c097151a4b270ee861 is 50, key is test_row_0/B:col10/1732514232681/Put/seqid=0 2024-11-25T05:57:13,885 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 39bebe481dd46e37e6dae001fcad1e53#C#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:13,886 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/c2c1cc9ac66e46b295a64feac4936aa9 is 50, key is test_row_0/C:col10/1732514231448/Put/seqid=0 2024-11-25T05:57:13,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742320_1496 (size=12151) 2024-11-25T05:57:13,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742321_1497 (size=12663) 2024-11-25T05:57:13,907 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/c2c1cc9ac66e46b295a64feac4936aa9 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/c2c1cc9ac66e46b295a64feac4936aa9 2024-11-25T05:57:13,911 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 39bebe481dd46e37e6dae001fcad1e53/C of 39bebe481dd46e37e6dae001fcad1e53 into c2c1cc9ac66e46b295a64feac4936aa9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:13,911 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:13,912 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., storeName=39bebe481dd46e37e6dae001fcad1e53/C, priority=13, startTime=1732514233840; duration=0sec 2024-11-25T05:57:13,912 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:13,912 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 39bebe481dd46e37e6dae001fcad1e53:C 2024-11-25T05:57:13,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T05:57:13,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:13,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:13,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:13,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:13,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:13,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:13,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:13,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514293962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:13,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:13,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:13,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:14,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T05:57:14,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:14,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:14,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:14,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:14,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:14,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:14,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:14,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514294082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:14,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:14,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:14,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:14,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:14,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34996 deadline: 1732514294171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:14,179 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_2' in 39bebe481dd46e37e6dae001fcad1e53 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:57:14,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:14,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:14,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:14,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:14,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514294182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:14,188 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_1' in 39bebe481dd46e37e6dae001fcad1e53 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:57:14,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:14,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:14,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:14,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:14,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34982 deadline: 1732514294182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:14,190 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_0' in 39bebe481dd46e37e6dae001fcad1e53 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:57:14,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:14,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:14,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:14,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:14,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35022 deadline: 1732514294212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:14,220 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_1' in 39bebe481dd46e37e6dae001fcad1e53 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:57:14,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:14,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:14,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:14,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T05:57:14,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:14,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:14,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:14,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:14,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:14,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:14,257 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/4b8886dcc45a4fcf891f3e184664e1ee as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/4b8886dcc45a4fcf891f3e184664e1ee 2024-11-25T05:57:14,260 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 39bebe481dd46e37e6dae001fcad1e53/A of 39bebe481dd46e37e6dae001fcad1e53 into 4b8886dcc45a4fcf891f3e184664e1ee(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:14,261 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:14,261 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., storeName=39bebe481dd46e37e6dae001fcad1e53/A, priority=13, startTime=1732514233840; duration=0sec 2024-11-25T05:57:14,261 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:14,261 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 39bebe481dd46e37e6dae001fcad1e53:A 2024-11-25T05:57:14,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/cca0c8c7c8de48c097151a4b270ee861 2024-11-25T05:57:14,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/d6948e9d6d604d4caa8dffbcae1a3b85 is 50, key is test_row_0/C:col10/1732514232681/Put/seqid=0 2024-11-25T05:57:14,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:14,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514294302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:14,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:14,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:14,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:14,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742322_1498 (size=12151) 2024-11-25T05:57:14,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T05:57:14,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:14,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:14,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:14,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:14,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:14,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:14,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T05:57:14,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:14,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:14,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:14,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:14,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:14,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:14,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:14,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514294612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:14,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:14,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:14,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:14,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T05:57:14,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:14,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:14,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:14,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:14,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:14,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:14,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/d6948e9d6d604d4caa8dffbcae1a3b85 2024-11-25T05:57:14,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/8a8f5d6a29c04c2394b2065f42ccea4a as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/8a8f5d6a29c04c2394b2065f42ccea4a 2024-11-25T05:57:14,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/8a8f5d6a29c04c2394b2065f42ccea4a, entries=200, sequenceid=219, filesize=14.2 K 2024-11-25T05:57:14,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/cca0c8c7c8de48c097151a4b270ee861 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/cca0c8c7c8de48c097151a4b270ee861 2024-11-25T05:57:14,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/cca0c8c7c8de48c097151a4b270ee861, entries=150, sequenceid=219, filesize=11.9 K 2024-11-25T05:57:14,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/d6948e9d6d604d4caa8dffbcae1a3b85 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/d6948e9d6d604d4caa8dffbcae1a3b85 2024-11-25T05:57:14,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/d6948e9d6d604d4caa8dffbcae1a3b85, entries=150, sequenceid=219, filesize=11.9 K 2024-11-25T05:57:14,724 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 39bebe481dd46e37e6dae001fcad1e53 in 868ms, sequenceid=219, compaction requested=false 2024-11-25T05:57:14,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:14,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T05:57:14,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:14,851 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 39bebe481dd46e37e6dae001fcad1e53 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-25T05:57:14,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=A 2024-11-25T05:57:14,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:14,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=B 2024-11-25T05:57:14,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:14,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=C 2024-11-25T05:57:14,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:14,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/d51bc294175d4083b975f4dcc825d06f is 50, key is test_row_0/A:col10/1732514233951/Put/seqid=0 2024-11-25T05:57:14,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742323_1499 (size=12151) 2024-11-25T05:57:14,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T05:57:15,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on 39bebe481dd46e37e6dae001fcad1e53 2024-11-25T05:57:15,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:15,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:15,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514295214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:15,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:15,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:15,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:15,263 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/d51bc294175d4083b975f4dcc825d06f 2024-11-25T05:57:15,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/aaf323065dd940788d3d318127e48fa9 is 50, key is test_row_0/B:col10/1732514233951/Put/seqid=0 2024-11-25T05:57:15,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742324_1500 (size=12151) 2024-11-25T05:57:15,276 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/aaf323065dd940788d3d318127e48fa9 2024-11-25T05:57:15,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/765febf546444668892069630b9edb34 is 50, key is test_row_0/C:col10/1732514233951/Put/seqid=0 2024-11-25T05:57:15,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742325_1501 (size=12151) 2024-11-25T05:57:15,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:15,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514295331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:15,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:15,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:15,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:15,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:15,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514295542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:15,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:15,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:15,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:15,690 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/765febf546444668892069630b9edb34 2024-11-25T05:57:15,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/d51bc294175d4083b975f4dcc825d06f as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/d51bc294175d4083b975f4dcc825d06f 2024-11-25T05:57:15,696 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/d51bc294175d4083b975f4dcc825d06f, entries=150, sequenceid=237, filesize=11.9 K 2024-11-25T05:57:15,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/aaf323065dd940788d3d318127e48fa9 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/aaf323065dd940788d3d318127e48fa9 2024-11-25T05:57:15,700 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/aaf323065dd940788d3d318127e48fa9, entries=150, sequenceid=237, filesize=11.9 K 2024-11-25T05:57:15,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/765febf546444668892069630b9edb34 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/765febf546444668892069630b9edb34 2024-11-25T05:57:15,704 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/765febf546444668892069630b9edb34, entries=150, sequenceid=237, filesize=11.9 K 2024-11-25T05:57:15,704 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 39bebe481dd46e37e6dae001fcad1e53 in 853ms, sequenceid=237, compaction requested=true 2024-11-25T05:57:15,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:15,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:15,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-25T05:57:15,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-25T05:57:15,707 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-25T05:57:15,707 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9900 sec 2024-11-25T05:57:15,708 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.9940 sec 2024-11-25T05:57:15,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on 39bebe481dd46e37e6dae001fcad1e53 2024-11-25T05:57:15,857 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 39bebe481dd46e37e6dae001fcad1e53 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-25T05:57:15,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=A 2024-11-25T05:57:15,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:15,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=B 2024-11-25T05:57:15,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:15,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=C 2024-11-25T05:57:15,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:15,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/8d3669e871074208b5faf4ab5231c10c is 50, key is test_row_0/A:col10/1732514235213/Put/seqid=0 2024-11-25T05:57:15,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742326_1502 (size=14541) 2024-11-25T05:57:15,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:15,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514295946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:15,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:15,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:15,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:16,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:16,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514296062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:16,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:16,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:16,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:16,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/8d3669e871074208b5faf4ab5231c10c 2024-11-25T05:57:16,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/090a19660aa84cb9a9e9070b1ff29236 is 50, key is test_row_0/B:col10/1732514235213/Put/seqid=0 2024-11-25T05:57:16,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742327_1503 (size=12151) 2024-11-25T05:57:16,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514296282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:16,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:16,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514296592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:16,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:16,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:16,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:16,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/090a19660aa84cb9a9e9070b1ff29236 2024-11-25T05:57:16,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/ab57f15d225845288bc7ce4ad2bbb169 is 50, key is test_row_0/C:col10/1732514235213/Put/seqid=0 2024-11-25T05:57:16,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742328_1504 (size=12151) 2024-11-25T05:57:16,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/ab57f15d225845288bc7ce4ad2bbb169 2024-11-25T05:57:16,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/8d3669e871074208b5faf4ab5231c10c as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/8d3669e871074208b5faf4ab5231c10c 2024-11-25T05:57:16,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/8d3669e871074208b5faf4ab5231c10c, entries=200, sequenceid=255, filesize=14.2 K 2024-11-25T05:57:16,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/090a19660aa84cb9a9e9070b1ff29236 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/090a19660aa84cb9a9e9070b1ff29236 2024-11-25T05:57:16,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/090a19660aa84cb9a9e9070b1ff29236, entries=150, sequenceid=255, filesize=11.9 K 2024-11-25T05:57:16,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/ab57f15d225845288bc7ce4ad2bbb169 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/ab57f15d225845288bc7ce4ad2bbb169 2024-11-25T05:57:16,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/ab57f15d225845288bc7ce4ad2bbb169, entries=150, sequenceid=255, filesize=11.9 K 2024-11-25T05:57:16,702 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 39bebe481dd46e37e6dae001fcad1e53 in 845ms, sequenceid=255, compaction requested=true 2024-11-25T05:57:16,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:16,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 39bebe481dd46e37e6dae001fcad1e53:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:57:16,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:16,703 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T05:57:16,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 39bebe481dd46e37e6dae001fcad1e53:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:57:16,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:16,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 39bebe481dd46e37e6dae001fcad1e53:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:57:16,703 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T05:57:16,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:57:16,703 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49116 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T05:57:16,703 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53896 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T05:57:16,703 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): 39bebe481dd46e37e6dae001fcad1e53/B is initiating minor compaction (all files) 2024-11-25T05:57:16,703 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): 39bebe481dd46e37e6dae001fcad1e53/A is initiating minor compaction (all files) 2024-11-25T05:57:16,703 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 39bebe481dd46e37e6dae001fcad1e53/B in TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:16,703 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 39bebe481dd46e37e6dae001fcad1e53/A in TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:16,704 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/4b8886dcc45a4fcf891f3e184664e1ee, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/8a8f5d6a29c04c2394b2065f42ccea4a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/d51bc294175d4083b975f4dcc825d06f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/8d3669e871074208b5faf4ab5231c10c] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp, totalSize=52.6 K 2024-11-25T05:57:16,704 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/1ba123900f4c429ba4299e19a0eabc2c, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/cca0c8c7c8de48c097151a4b270ee861, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/aaf323065dd940788d3d318127e48fa9, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/090a19660aa84cb9a9e9070b1ff29236] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp, totalSize=48.0 K 2024-11-25T05:57:16,704 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ba123900f4c429ba4299e19a0eabc2c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732514231416 2024-11-25T05:57:16,704 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b8886dcc45a4fcf891f3e184664e1ee, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732514231416 2024-11-25T05:57:16,704 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting cca0c8c7c8de48c097151a4b270ee861, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732514232665 2024-11-25T05:57:16,704 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a8f5d6a29c04c2394b2065f42ccea4a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732514232665 2024-11-25T05:57:16,704 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting aaf323065dd940788d3d318127e48fa9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732514233928 2024-11-25T05:57:16,704 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting d51bc294175d4083b975f4dcc825d06f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732514233928 2024-11-25T05:57:16,704 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 090a19660aa84cb9a9e9070b1ff29236, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732514235195 2024-11-25T05:57:16,704 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d3669e871074208b5faf4ab5231c10c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732514235155 2024-11-25T05:57:16,710 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 39bebe481dd46e37e6dae001fcad1e53#B#compaction#434 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:16,710 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/7684246350e74c96a7a96dcfb5cdea8c is 50, key is test_row_0/B:col10/1732514235213/Put/seqid=0 2024-11-25T05:57:16,713 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 39bebe481dd46e37e6dae001fcad1e53#A#compaction#435 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:16,714 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/4d90f864724d438dafea00f5b7e6b1cd is 50, key is test_row_0/A:col10/1732514235213/Put/seqid=0 2024-11-25T05:57:16,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742329_1505 (size=12799) 2024-11-25T05:57:16,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742330_1506 (size=12799) 2024-11-25T05:57:16,728 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/4d90f864724d438dafea00f5b7e6b1cd as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/4d90f864724d438dafea00f5b7e6b1cd 2024-11-25T05:57:16,732 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 39bebe481dd46e37e6dae001fcad1e53/A of 39bebe481dd46e37e6dae001fcad1e53 into 4d90f864724d438dafea00f5b7e6b1cd(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:16,732 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:16,732 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., storeName=39bebe481dd46e37e6dae001fcad1e53/A, priority=12, startTime=1732514236702; duration=0sec 2024-11-25T05:57:16,733 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:57:16,733 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 39bebe481dd46e37e6dae001fcad1e53:A 2024-11-25T05:57:16,733 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T05:57:16,734 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49116 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T05:57:16,734 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): 39bebe481dd46e37e6dae001fcad1e53/C is initiating minor compaction (all files) 2024-11-25T05:57:16,734 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 39bebe481dd46e37e6dae001fcad1e53/C in TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:16,734 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/c2c1cc9ac66e46b295a64feac4936aa9, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/d6948e9d6d604d4caa8dffbcae1a3b85, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/765febf546444668892069630b9edb34, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/ab57f15d225845288bc7ce4ad2bbb169] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp, totalSize=48.0 K 2024-11-25T05:57:16,734 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2c1cc9ac66e46b295a64feac4936aa9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732514231416 2024-11-25T05:57:16,734 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6948e9d6d604d4caa8dffbcae1a3b85, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732514232665 2024-11-25T05:57:16,734 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 765febf546444668892069630b9edb34, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732514233928 2024-11-25T05:57:16,735 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab57f15d225845288bc7ce4ad2bbb169, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732514235195 2024-11-25T05:57:16,741 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 39bebe481dd46e37e6dae001fcad1e53#C#compaction#436 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:16,742 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/3d45999ff82a47b5b500353ddde1fb68 is 50, key is test_row_0/C:col10/1732514235213/Put/seqid=0 2024-11-25T05:57:16,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742331_1507 (size=12799) 2024-11-25T05:57:16,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T05:57:16,883 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:57:16,883 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:57:16,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-25T05:57:16,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-25T05:57:16,884 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:57:16,885 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:57:16,885 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:57:16,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-25T05:57:17,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-25T05:57:17,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:17,037 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 39bebe481dd46e37e6dae001fcad1e53 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-25T05:57:17,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=A 2024-11-25T05:57:17,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:17,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=B 2024-11-25T05:57:17,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:17,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=C 2024-11-25T05:57:17,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:17,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/a7882b6bf7a845df9a57c0d93ccc8972 is 50, key is test_row_0/A:col10/1732514235932/Put/seqid=0 2024-11-25T05:57:17,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742332_1508 (size=12301) 2024-11-25T05:57:17,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:17,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on 39bebe481dd46e37e6dae001fcad1e53 2024-11-25T05:57:17,121 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/7684246350e74c96a7a96dcfb5cdea8c as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/7684246350e74c96a7a96dcfb5cdea8c 2024-11-25T05:57:17,125 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 39bebe481dd46e37e6dae001fcad1e53/B of 39bebe481dd46e37e6dae001fcad1e53 into 7684246350e74c96a7a96dcfb5cdea8c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:17,126 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:17,126 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., storeName=39bebe481dd46e37e6dae001fcad1e53/B, priority=12, startTime=1732514236703; duration=0sec 2024-11-25T05:57:17,126 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:17,126 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 39bebe481dd46e37e6dae001fcad1e53:B 2024-11-25T05:57:17,149 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/3d45999ff82a47b5b500353ddde1fb68 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/3d45999ff82a47b5b500353ddde1fb68 2024-11-25T05:57:17,152 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 39bebe481dd46e37e6dae001fcad1e53/C of 39bebe481dd46e37e6dae001fcad1e53 into 3d45999ff82a47b5b500353ddde1fb68(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:17,152 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:17,152 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., storeName=39bebe481dd46e37e6dae001fcad1e53/C, priority=12, startTime=1732514236703; duration=0sec 2024-11-25T05:57:17,152 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:17,152 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 39bebe481dd46e37e6dae001fcad1e53:C 2024-11-25T05:57:17,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-25T05:57:17,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:17,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514297213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:17,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:17,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:17,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:17,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:17,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514297322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:17,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:17,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:17,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:17,444 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/a7882b6bf7a845df9a57c0d93ccc8972 2024-11-25T05:57:17,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/07fabc318e8a4065a1cdd7b09fc75165 is 50, key is test_row_0/B:col10/1732514235932/Put/seqid=0 2024-11-25T05:57:17,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742333_1509 (size=12301) 2024-11-25T05:57:17,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-25T05:57:17,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:17,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514297532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:17,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:17,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:17,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:17,546 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T05:57:17,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:17,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514297842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:17,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:17,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:17,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:17,855 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/07fabc318e8a4065a1cdd7b09fc75165 2024-11-25T05:57:17,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/d53ae1a5679b4fd2a34d33e5405bf8df is 50, key is test_row_0/C:col10/1732514235932/Put/seqid=0 2024-11-25T05:57:17,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742334_1510 (size=12301) 2024-11-25T05:57:18,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-25T05:57:18,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:18,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34976 deadline: 1732514298202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:18,207 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_1' in 39bebe481dd46e37e6dae001fcad1e53 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:57:18,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:18,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:18,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:18,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34996 deadline: 1732514298212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:18,218 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_2' in 39bebe481dd46e37e6dae001fcad1e53 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:57:18,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:18,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:18,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:18,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:18,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34982 deadline: 1732514298232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:18,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35022 deadline: 1732514298232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:18,241 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_0' in 39bebe481dd46e37e6dae001fcad1e53 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8258 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:57:18,241 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_1' in 39bebe481dd46e37e6dae001fcad1e53 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:57:18,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:18,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:18,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:18,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:18,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:18,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:18,267 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/d53ae1a5679b4fd2a34d33e5405bf8df 2024-11-25T05:57:18,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/a7882b6bf7a845df9a57c0d93ccc8972 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/a7882b6bf7a845df9a57c0d93ccc8972 2024-11-25T05:57:18,274 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/a7882b6bf7a845df9a57c0d93ccc8972, entries=150, sequenceid=270, filesize=12.0 K 2024-11-25T05:57:18,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/07fabc318e8a4065a1cdd7b09fc75165 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/07fabc318e8a4065a1cdd7b09fc75165 2024-11-25T05:57:18,277 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/07fabc318e8a4065a1cdd7b09fc75165, entries=150, sequenceid=270, filesize=12.0 K 2024-11-25T05:57:18,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/d53ae1a5679b4fd2a34d33e5405bf8df as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/d53ae1a5679b4fd2a34d33e5405bf8df 2024-11-25T05:57:18,283 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/d53ae1a5679b4fd2a34d33e5405bf8df, entries=150, sequenceid=270, filesize=12.0 K 2024-11-25T05:57:18,284 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 39bebe481dd46e37e6dae001fcad1e53 in 1247ms, sequenceid=270, compaction requested=false 2024-11-25T05:57:18,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:18,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:18,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-25T05:57:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-25T05:57:18,287 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-25T05:57:18,287 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4000 sec 2024-11-25T05:57:18,288 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.4040 sec 2024-11-25T05:57:18,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on 39bebe481dd46e37e6dae001fcad1e53 2024-11-25T05:57:18,366 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 39bebe481dd46e37e6dae001fcad1e53 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-25T05:57:18,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=A 2024-11-25T05:57:18,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:18,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=B 2024-11-25T05:57:18,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:18,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=C 2024-11-25T05:57:18,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:18,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/2b2bec46685140918420489cec425f2e is 50, key is test_row_0/A:col10/1732514238365/Put/seqid=0 2024-11-25T05:57:18,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742335_1511 (size=14741) 2024-11-25T05:57:18,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:18,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514298423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:18,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:18,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:18,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:18,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:18,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514298542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:18,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:18,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:18,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:18,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:18,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514298752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:18,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:18,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:18,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:18,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/2b2bec46685140918420489cec425f2e 2024-11-25T05:57:18,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/83b22229f1c44dfd85b8c09b957bc99e is 50, key is test_row_0/B:col10/1732514238365/Put/seqid=0 2024-11-25T05:57:18,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742336_1512 (size=12301) 2024-11-25T05:57:19,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-25T05:57:19,032 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:57:19,033 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:57:19,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-25T05:57:19,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-25T05:57:19,034 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:57:19,035 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:57:19,035 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:57:19,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:19,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514299061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:19,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:19,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:19,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:19,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-25T05:57:19,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T05:57:19,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:19,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:19,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:19,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:19,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:19,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:19,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/83b22229f1c44dfd85b8c09b957bc99e 2024-11-25T05:57:19,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/397967ce0d7c4c07ad6ebecd04eedfc8 is 50, key is test_row_0/C:col10/1732514238365/Put/seqid=0 2024-11-25T05:57:19,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742337_1513 (size=12301) 2024-11-25T05:57:19,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T05:57:19,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:19,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:19,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:19,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:19,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:19,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:19,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-25T05:57:19,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T05:57:19,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:19,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:19,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:19,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:19,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:19,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:19,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:19,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732514299582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:19,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:19,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:19,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., hostname=8ef925b832e3,35599,1732514089005, seqNum=2 because the exception is null or not the one we care about 2024-11-25T05:57:19,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/397967ce0d7c4c07ad6ebecd04eedfc8 2024-11-25T05:57:19,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/2b2bec46685140918420489cec425f2e as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/2b2bec46685140918420489cec425f2e 2024-11-25T05:57:19,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/2b2bec46685140918420489cec425f2e, entries=200, sequenceid=291, filesize=14.4 K 2024-11-25T05:57:19,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/83b22229f1c44dfd85b8c09b957bc99e as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/83b22229f1c44dfd85b8c09b957bc99e 2024-11-25T05:57:19,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/83b22229f1c44dfd85b8c09b957bc99e, entries=150, sequenceid=291, filesize=12.0 K 2024-11-25T05:57:19,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/397967ce0d7c4c07ad6ebecd04eedfc8 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/397967ce0d7c4c07ad6ebecd04eedfc8 2024-11-25T05:57:19,618 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/397967ce0d7c4c07ad6ebecd04eedfc8, entries=150, sequenceid=291, filesize=12.0 K 2024-11-25T05:57:19,619 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 39bebe481dd46e37e6dae001fcad1e53 in 1253ms, sequenceid=291, compaction requested=true 2024-11-25T05:57:19,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:19,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 39bebe481dd46e37e6dae001fcad1e53:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:57:19,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:19,619 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:57:19,619 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:57:19,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 39bebe481dd46e37e6dae001fcad1e53:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:57:19,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:19,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 39bebe481dd46e37e6dae001fcad1e53:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:57:19,620 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39841 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:57:19,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:57:19,620 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): 39bebe481dd46e37e6dae001fcad1e53/A is initiating minor compaction (all files) 2024-11-25T05:57:19,620 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 39bebe481dd46e37e6dae001fcad1e53/A in TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:19,620 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/4d90f864724d438dafea00f5b7e6b1cd, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/a7882b6bf7a845df9a57c0d93ccc8972, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/2b2bec46685140918420489cec425f2e] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp, totalSize=38.9 K 2024-11-25T05:57:19,620 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:57:19,620 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): 39bebe481dd46e37e6dae001fcad1e53/B is initiating minor compaction (all files) 2024-11-25T05:57:19,620 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 39bebe481dd46e37e6dae001fcad1e53/B in TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:19,620 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/7684246350e74c96a7a96dcfb5cdea8c, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/07fabc318e8a4065a1cdd7b09fc75165, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/83b22229f1c44dfd85b8c09b957bc99e] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp, totalSize=36.5 K 2024-11-25T05:57:19,621 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 7684246350e74c96a7a96dcfb5cdea8c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732514235195 2024-11-25T05:57:19,621 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d90f864724d438dafea00f5b7e6b1cd, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732514235195 2024-11-25T05:57:19,621 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 07fabc318e8a4065a1cdd7b09fc75165, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732514235925 2024-11-25T05:57:19,621 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7882b6bf7a845df9a57c0d93ccc8972, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732514235925 2024-11-25T05:57:19,621 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 83b22229f1c44dfd85b8c09b957bc99e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732514237127 2024-11-25T05:57:19,621 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b2bec46685140918420489cec425f2e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732514237127 2024-11-25T05:57:19,626 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 39bebe481dd46e37e6dae001fcad1e53#B#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:19,626 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 39bebe481dd46e37e6dae001fcad1e53#A#compaction#443 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:19,626 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/c4a4ebb0b0ca4524ae728e6ec405c35c is 50, key is test_row_0/B:col10/1732514238365/Put/seqid=0 2024-11-25T05:57:19,626 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/02b3361d085648e09d067fe98dc56a1e is 50, key is test_row_0/A:col10/1732514238365/Put/seqid=0 2024-11-25T05:57:19,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742339_1515 (size=13051) 2024-11-25T05:57:19,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742338_1514 (size=13051) 2024-11-25T05:57:19,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T05:57:19,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:19,645 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 39bebe481dd46e37e6dae001fcad1e53 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-25T05:57:19,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=A 2024-11-25T05:57:19,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:19,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=B 2024-11-25T05:57:19,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:19,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=C 2024-11-25T05:57:19,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:19,648 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/02b3361d085648e09d067fe98dc56a1e as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/02b3361d085648e09d067fe98dc56a1e 2024-11-25T05:57:19,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/66b98b87c83c4e58b56b38805ee1886b is 50, key is test_row_0/A:col10/1732514238422/Put/seqid=0 2024-11-25T05:57:19,655 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 39bebe481dd46e37e6dae001fcad1e53/A of 39bebe481dd46e37e6dae001fcad1e53 into 02b3361d085648e09d067fe98dc56a1e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:19,655 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:19,655 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., storeName=39bebe481dd46e37e6dae001fcad1e53/A, priority=13, startTime=1732514239619; duration=0sec 2024-11-25T05:57:19,655 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:57:19,655 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 39bebe481dd46e37e6dae001fcad1e53:A 2024-11-25T05:57:19,655 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:57:19,655 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:57:19,656 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): 39bebe481dd46e37e6dae001fcad1e53/C is initiating minor compaction (all files) 2024-11-25T05:57:19,656 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 39bebe481dd46e37e6dae001fcad1e53/C in TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:19,656 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/3d45999ff82a47b5b500353ddde1fb68, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/d53ae1a5679b4fd2a34d33e5405bf8df, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/397967ce0d7c4c07ad6ebecd04eedfc8] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp, totalSize=36.5 K 2024-11-25T05:57:19,656 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d45999ff82a47b5b500353ddde1fb68, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732514235195 2024-11-25T05:57:19,656 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting d53ae1a5679b4fd2a34d33e5405bf8df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732514235925 2024-11-25T05:57:19,656 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 397967ce0d7c4c07ad6ebecd04eedfc8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732514237127 2024-11-25T05:57:19,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-25T05:57:19,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742340_1516 (size=12301) 2024-11-25T05:57:19,664 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 39bebe481dd46e37e6dae001fcad1e53#C#compaction#446 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:19,664 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/f1c6f733969b4401ba42c915f97bbeb0 is 50, key is test_row_0/C:col10/1732514238365/Put/seqid=0 2024-11-25T05:57:19,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742341_1517 (size=13051) 2024-11-25T05:57:19,954 INFO [Thread-2019 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2019. 2024-11-25T05:57:19,954 DEBUG [Thread-2019 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:57:19,954 DEBUG [Thread-2019 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:19,954 DEBUG [Thread-2019 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:19,954 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:57:19,955 INFO [Thread-2013 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2013. 2024-11-25T05:57:19,955 DEBUG [Thread-2013 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:57:19,955 DEBUG [Thread-2013 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:19,955 DEBUG [Thread-2013 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:19,955 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:57:19,960 INFO [Thread-2015 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2015. 2024-11-25T05:57:19,960 DEBUG [Thread-2015 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:57:19,960 DEBUG [Thread-2015 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:19,960 DEBUG [Thread-2015 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:19,960 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:57:19,960 INFO [Thread-2011 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2011. 2024-11-25T05:57:19,960 DEBUG [Thread-2011 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:57:19,960 DEBUG [Thread-2011 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:19,960 DEBUG [Thread-2011 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:19,960 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:57:19,962 INFO [Thread-2017 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2017. 2024-11-25T05:57:19,962 DEBUG [Thread-2017 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:57:19,962 DEBUG [Thread-2017 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:19,962 DEBUG [Thread-2017 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:19,962 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:57:20,051 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/c4a4ebb0b0ca4524ae728e6ec405c35c as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/c4a4ebb0b0ca4524ae728e6ec405c35c 2024-11-25T05:57:20,056 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 39bebe481dd46e37e6dae001fcad1e53/B of 39bebe481dd46e37e6dae001fcad1e53 into c4a4ebb0b0ca4524ae728e6ec405c35c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:20,056 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:20,056 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., storeName=39bebe481dd46e37e6dae001fcad1e53/B, priority=13, startTime=1732514239619; duration=0sec 2024-11-25T05:57:20,056 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:20,056 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 39bebe481dd46e37e6dae001fcad1e53:B 2024-11-25T05:57:20,063 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/66b98b87c83c4e58b56b38805ee1886b 2024-11-25T05:57:20,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/a8e4db9beb5f4b2e9dd57437b72768f9 is 50, key is test_row_0/B:col10/1732514238422/Put/seqid=0 2024-11-25T05:57:20,071 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/f1c6f733969b4401ba42c915f97bbeb0 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/f1c6f733969b4401ba42c915f97bbeb0 2024-11-25T05:57:20,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742342_1518 (size=12301) 2024-11-25T05:57:20,074 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 39bebe481dd46e37e6dae001fcad1e53/C of 39bebe481dd46e37e6dae001fcad1e53 into f1c6f733969b4401ba42c915f97bbeb0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:20,074 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:20,074 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., storeName=39bebe481dd46e37e6dae001fcad1e53/C, priority=13, startTime=1732514239620; duration=0sec 2024-11-25T05:57:20,074 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:20,074 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 39bebe481dd46e37e6dae001fcad1e53:C 2024-11-25T05:57:20,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-25T05:57:20,473 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/a8e4db9beb5f4b2e9dd57437b72768f9 2024-11-25T05:57:20,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/03eedc3a8d654633bf50bc0be4692fd5 is 50, key is test_row_0/C:col10/1732514238422/Put/seqid=0 2024-11-25T05:57:20,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742343_1519 (size=12301) 2024-11-25T05:57:20,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on 39bebe481dd46e37e6dae001fcad1e53 2024-11-25T05:57:20,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. as already flushing 2024-11-25T05:57:20,604 INFO [Thread-2000 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2000. 2024-11-25T05:57:20,604 DEBUG [Thread-2000 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:57:20,604 DEBUG [Thread-2000 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:20,604 DEBUG [Thread-2000 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:20,604 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:57:20,889 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/03eedc3a8d654633bf50bc0be4692fd5 2024-11-25T05:57:20,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/66b98b87c83c4e58b56b38805ee1886b as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/66b98b87c83c4e58b56b38805ee1886b 2024-11-25T05:57:20,903 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/66b98b87c83c4e58b56b38805ee1886b, entries=150, sequenceid=304, filesize=12.0 K 2024-11-25T05:57:20,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/a8e4db9beb5f4b2e9dd57437b72768f9 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/a8e4db9beb5f4b2e9dd57437b72768f9 2024-11-25T05:57:20,908 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/a8e4db9beb5f4b2e9dd57437b72768f9, entries=150, sequenceid=304, filesize=12.0 K 2024-11-25T05:57:20,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/03eedc3a8d654633bf50bc0be4692fd5 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/03eedc3a8d654633bf50bc0be4692fd5 2024-11-25T05:57:20,911 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/03eedc3a8d654633bf50bc0be4692fd5, entries=150, sequenceid=304, filesize=12.0 K 2024-11-25T05:57:20,912 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=6.71 KB/6870 for 39bebe481dd46e37e6dae001fcad1e53 in 1267ms, sequenceid=304, compaction requested=false 2024-11-25T05:57:20,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 39bebe481dd46e37e6dae001fcad1e53: 2024-11-25T05:57:20,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:20,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-25T05:57:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-25T05:57:20,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-25T05:57:20,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8780 sec 2024-11-25T05:57:20,916 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.8820 sec 2024-11-25T05:57:21,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-25T05:57:21,183 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:57:28,276 INFO [Thread-2008 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2008. 2024-11-25T05:57:28,276 DEBUG [Thread-2008 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:57:28,276 DEBUG [Thread-2008 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:28,277 DEBUG [Thread-2008 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:28,277 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:57:28,295 INFO [Thread-2004 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2004. 2024-11-25T05:57:28,295 DEBUG [Thread-2004 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:57:28,295 DEBUG [Thread-2004 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:28,295 DEBUG [Thread-2004 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:28,295 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:57:28,326 INFO [Thread-2006 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2006. 2024-11-25T05:57:28,326 DEBUG [Thread-2006 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:57:28,327 DEBUG [Thread-2006 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:28,327 DEBUG [Thread-2006 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:28,327 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:57:28,345 INFO [Thread-2002 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2002. 2024-11-25T05:57:28,345 DEBUG [Thread-2002 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:57:28,345 DEBUG [Thread-2002 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:28,345 DEBUG [Thread-2002 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:28,345 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:57:28,346 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-25T05:57:28,346 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 114 2024-11-25T05:57:28,346 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-11-25T05:57:28,346 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-25T05:57:28,346 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-11-25T05:57:28,346 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-25T05:57:28,346 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-25T05:57:28,346 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-25T05:57:28,346 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1228 2024-11-25T05:57:28,346 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3683 rows 2024-11-25T05:57:28,347 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1237 2024-11-25T05:57:28,347 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3709 rows 2024-11-25T05:57:28,347 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1236 2024-11-25T05:57:28,347 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3704 rows 2024-11-25T05:57:28,347 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1231 2024-11-25T05:57:28,347 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3693 rows 2024-11-25T05:57:28,347 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1241 2024-11-25T05:57:28,347 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3716 rows 2024-11-25T05:57:28,347 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T05:57:28,347 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T05:57:28,347 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:28,348 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:28,348 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:57:28,349 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-25T05:57:28,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-25T05:57:28,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-25T05:57:28,353 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732514248353"}]},"ts":"1732514248353"} 2024-11-25T05:57:28,354 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-25T05:57:28,354 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-25T05:57:28,355 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T05:57:28,356 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=39bebe481dd46e37e6dae001fcad1e53, UNASSIGN}] 2024-11-25T05:57:28,357 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=39bebe481dd46e37e6dae001fcad1e53, UNASSIGN 2024-11-25T05:57:28,358 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=127 updating hbase:meta row=39bebe481dd46e37e6dae001fcad1e53, regionState=CLOSING, regionLocation=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:28,359 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=39bebe481dd46e37e6dae001fcad1e53, UNASSIGN because future has completed 2024-11-25T05:57:28,359 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T05:57:28,359 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; CloseRegionProcedure 39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005}] 2024-11-25T05:57:28,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-25T05:57:28,512 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] handler.UnassignRegionHandler(122): Close 39bebe481dd46e37e6dae001fcad1e53 2024-11-25T05:57:28,512 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T05:57:28,512 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1722): Closing 39bebe481dd46e37e6dae001fcad1e53, disabling compactions & flushes 2024-11-25T05:57:28,512 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:28,512 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:28,512 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. after waiting 0 ms 2024-11-25T05:57:28,512 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:28,512 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(2902): Flushing 39bebe481dd46e37e6dae001fcad1e53 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-25T05:57:28,513 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=A 2024-11-25T05:57:28,513 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:28,513 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=B 2024-11-25T05:57:28,513 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:28,513 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 39bebe481dd46e37e6dae001fcad1e53, store=C 2024-11-25T05:57:28,513 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:28,518 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/26e6714720dc4839affe92848e04a2d1 is 50, key is test_row_0/A:col10/1732514248323/Put/seqid=0 2024-11-25T05:57:28,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742344_1520 (size=12301) 2024-11-25T05:57:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-25T05:57:28,924 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/26e6714720dc4839affe92848e04a2d1 2024-11-25T05:57:28,937 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/43041955311542da8b7e5a2e7044884c is 50, key is test_row_0/B:col10/1732514248323/Put/seqid=0 2024-11-25T05:57:28,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742345_1521 (size=12301) 2024-11-25T05:57:28,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-25T05:57:29,342 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/43041955311542da8b7e5a2e7044884c 2024-11-25T05:57:29,351 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/817064493e58452fb01a8e825be1d2eb is 50, key is test_row_0/C:col10/1732514248323/Put/seqid=0 2024-11-25T05:57:29,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742346_1522 (size=12301) 2024-11-25T05:57:29,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-25T05:57:29,757 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/817064493e58452fb01a8e825be1d2eb 2024-11-25T05:57:29,766 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/A/26e6714720dc4839affe92848e04a2d1 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/26e6714720dc4839affe92848e04a2d1 2024-11-25T05:57:29,772 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/26e6714720dc4839affe92848e04a2d1, entries=150, sequenceid=315, filesize=12.0 K 2024-11-25T05:57:29,773 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/B/43041955311542da8b7e5a2e7044884c as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/43041955311542da8b7e5a2e7044884c 2024-11-25T05:57:29,777 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/43041955311542da8b7e5a2e7044884c, entries=150, sequenceid=315, filesize=12.0 K 2024-11-25T05:57:29,777 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/.tmp/C/817064493e58452fb01a8e825be1d2eb as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/817064493e58452fb01a8e825be1d2eb 2024-11-25T05:57:29,781 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/817064493e58452fb01a8e825be1d2eb, entries=150, sequenceid=315, filesize=12.0 K 2024-11-25T05:57:29,782 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 39bebe481dd46e37e6dae001fcad1e53 in 1270ms, sequenceid=315, compaction requested=true 2024-11-25T05:57:29,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/c8281def41da4402b0b038bcaaddb1f6, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/59971285dc6f49e49e2ca4fa13965541, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/682ddbd0a47247eb85bc598865e49c8f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/7e984871f62e4f86af09a902842582fa, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/a528ba6761ea4819ab4d38404c73416a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/7f003ab58d4646969f586fa5d8f97876, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/acd48d58142e48b4a87e27b6e263706a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/8ba4f1610dfd4a73998ad90598066da1, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/44e3e7c4f2d44323a073c80906ec7225, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/21298ef611ef470a804c3173fdf49a6f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/1150a10764d54af3a02df9f6401cada4, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/6744ee50790849e390c927a73d295d72, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/4d0e685c0e2c4bb3bc2bffff7bdf5597, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/48dd514cbf1b468397e28d93fcabf2a9, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/b55a31378a6a42bea5c9b083b7f43bdf, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/4b8886dcc45a4fcf891f3e184664e1ee, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/8a8f5d6a29c04c2394b2065f42ccea4a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/d51bc294175d4083b975f4dcc825d06f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/8d3669e871074208b5faf4ab5231c10c, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/4d90f864724d438dafea00f5b7e6b1cd, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/a7882b6bf7a845df9a57c0d93ccc8972, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/2b2bec46685140918420489cec425f2e] to archive 2024-11-25T05:57:29,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T05:57:29,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/c8281def41da4402b0b038bcaaddb1f6 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/c8281def41da4402b0b038bcaaddb1f6 2024-11-25T05:57:29,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/59971285dc6f49e49e2ca4fa13965541 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/59971285dc6f49e49e2ca4fa13965541 2024-11-25T05:57:29,787 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/682ddbd0a47247eb85bc598865e49c8f to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/682ddbd0a47247eb85bc598865e49c8f 2024-11-25T05:57:29,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/7e984871f62e4f86af09a902842582fa to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/7e984871f62e4f86af09a902842582fa 2024-11-25T05:57:29,790 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/a528ba6761ea4819ab4d38404c73416a to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/a528ba6761ea4819ab4d38404c73416a 2024-11-25T05:57:29,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/7f003ab58d4646969f586fa5d8f97876 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/7f003ab58d4646969f586fa5d8f97876 2024-11-25T05:57:29,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/acd48d58142e48b4a87e27b6e263706a to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/acd48d58142e48b4a87e27b6e263706a 2024-11-25T05:57:29,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/8ba4f1610dfd4a73998ad90598066da1 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/8ba4f1610dfd4a73998ad90598066da1 2024-11-25T05:57:29,794 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/44e3e7c4f2d44323a073c80906ec7225 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/44e3e7c4f2d44323a073c80906ec7225 2024-11-25T05:57:29,796 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/21298ef611ef470a804c3173fdf49a6f to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/21298ef611ef470a804c3173fdf49a6f 2024-11-25T05:57:29,797 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/1150a10764d54af3a02df9f6401cada4 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/1150a10764d54af3a02df9f6401cada4 2024-11-25T05:57:29,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/6744ee50790849e390c927a73d295d72 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/6744ee50790849e390c927a73d295d72 2024-11-25T05:57:29,800 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/4d0e685c0e2c4bb3bc2bffff7bdf5597 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/4d0e685c0e2c4bb3bc2bffff7bdf5597 2024-11-25T05:57:29,801 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/48dd514cbf1b468397e28d93fcabf2a9 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/48dd514cbf1b468397e28d93fcabf2a9 2024-11-25T05:57:29,803 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/b55a31378a6a42bea5c9b083b7f43bdf to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/b55a31378a6a42bea5c9b083b7f43bdf 2024-11-25T05:57:29,804 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/4b8886dcc45a4fcf891f3e184664e1ee to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/4b8886dcc45a4fcf891f3e184664e1ee 2024-11-25T05:57:29,806 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/8a8f5d6a29c04c2394b2065f42ccea4a to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/8a8f5d6a29c04c2394b2065f42ccea4a 2024-11-25T05:57:29,807 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/d51bc294175d4083b975f4dcc825d06f to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/d51bc294175d4083b975f4dcc825d06f 2024-11-25T05:57:29,808 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/8d3669e871074208b5faf4ab5231c10c to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/8d3669e871074208b5faf4ab5231c10c 2024-11-25T05:57:29,810 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/4d90f864724d438dafea00f5b7e6b1cd to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/4d90f864724d438dafea00f5b7e6b1cd 2024-11-25T05:57:29,811 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/a7882b6bf7a845df9a57c0d93ccc8972 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/a7882b6bf7a845df9a57c0d93ccc8972 2024-11-25T05:57:29,813 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/2b2bec46685140918420489cec425f2e to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/2b2bec46685140918420489cec425f2e 2024-11-25T05:57:29,815 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/fbe09a23208b4bc2aa126d7b9c36a238, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/ba01b74530b9405188eacce70f28efee, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/706de24bb95c4636be0d45705f19a869, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/a4be286995394065923978ff3d0d040b, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/72bee1f7a6e74beaba1d9dbee199ee76, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/ce80ea2cc39a4b73baa5c0ccf1cdedf3, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/8f5a9f1ab3ce47a5aa5c1bfa62067d0b, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/8e51ca1b06d54ce0b0026bf179980f4e, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/b7fa76a0f7004eee8977da15bb98ecfb, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/4a87c71333284cc08829b47a4b930ae8, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/216bb1892c3648f0895c8b237909ba58, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/5f15fbabb64f4298acbaf13ac9a36f89, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/1ded07da516b40ffb0561e68eacd8fe8, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/32567a00419841acad70aaa1edfaa17e, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/1ba123900f4c429ba4299e19a0eabc2c, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/8cff78c375a848af8cb423da66f1e630, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/cca0c8c7c8de48c097151a4b270ee861, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/aaf323065dd940788d3d318127e48fa9, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/7684246350e74c96a7a96dcfb5cdea8c, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/090a19660aa84cb9a9e9070b1ff29236, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/07fabc318e8a4065a1cdd7b09fc75165, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/83b22229f1c44dfd85b8c09b957bc99e] to archive 2024-11-25T05:57:29,816 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T05:57:29,818 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/fbe09a23208b4bc2aa126d7b9c36a238 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/fbe09a23208b4bc2aa126d7b9c36a238 2024-11-25T05:57:29,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/ba01b74530b9405188eacce70f28efee to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/ba01b74530b9405188eacce70f28efee 2024-11-25T05:57:29,821 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/706de24bb95c4636be0d45705f19a869 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/706de24bb95c4636be0d45705f19a869 2024-11-25T05:57:29,823 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/a4be286995394065923978ff3d0d040b to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/a4be286995394065923978ff3d0d040b 2024-11-25T05:57:29,824 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/72bee1f7a6e74beaba1d9dbee199ee76 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/72bee1f7a6e74beaba1d9dbee199ee76 2024-11-25T05:57:29,825 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/ce80ea2cc39a4b73baa5c0ccf1cdedf3 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/ce80ea2cc39a4b73baa5c0ccf1cdedf3 2024-11-25T05:57:29,827 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/8f5a9f1ab3ce47a5aa5c1bfa62067d0b to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/8f5a9f1ab3ce47a5aa5c1bfa62067d0b 2024-11-25T05:57:29,828 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/8e51ca1b06d54ce0b0026bf179980f4e to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/8e51ca1b06d54ce0b0026bf179980f4e 2024-11-25T05:57:29,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/b7fa76a0f7004eee8977da15bb98ecfb to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/b7fa76a0f7004eee8977da15bb98ecfb 2024-11-25T05:57:29,831 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/4a87c71333284cc08829b47a4b930ae8 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/4a87c71333284cc08829b47a4b930ae8 2024-11-25T05:57:29,832 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/216bb1892c3648f0895c8b237909ba58 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/216bb1892c3648f0895c8b237909ba58 2024-11-25T05:57:29,834 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/5f15fbabb64f4298acbaf13ac9a36f89 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/5f15fbabb64f4298acbaf13ac9a36f89 2024-11-25T05:57:29,835 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/1ded07da516b40ffb0561e68eacd8fe8 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/1ded07da516b40ffb0561e68eacd8fe8 2024-11-25T05:57:29,837 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/32567a00419841acad70aaa1edfaa17e to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/32567a00419841acad70aaa1edfaa17e 2024-11-25T05:57:29,839 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/1ba123900f4c429ba4299e19a0eabc2c to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/1ba123900f4c429ba4299e19a0eabc2c 2024-11-25T05:57:29,840 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/8cff78c375a848af8cb423da66f1e630 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/8cff78c375a848af8cb423da66f1e630 2024-11-25T05:57:29,841 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/cca0c8c7c8de48c097151a4b270ee861 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/cca0c8c7c8de48c097151a4b270ee861 2024-11-25T05:57:29,842 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/aaf323065dd940788d3d318127e48fa9 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/aaf323065dd940788d3d318127e48fa9 2024-11-25T05:57:29,844 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/7684246350e74c96a7a96dcfb5cdea8c to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/7684246350e74c96a7a96dcfb5cdea8c 2024-11-25T05:57:29,845 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/090a19660aa84cb9a9e9070b1ff29236 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/090a19660aa84cb9a9e9070b1ff29236 2024-11-25T05:57:29,846 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/07fabc318e8a4065a1cdd7b09fc75165 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/07fabc318e8a4065a1cdd7b09fc75165 2024-11-25T05:57:29,847 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/83b22229f1c44dfd85b8c09b957bc99e to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/83b22229f1c44dfd85b8c09b957bc99e 2024-11-25T05:57:29,849 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/4ff6f500204c4769bcddc48c412a3115, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/d866dfc1e036418dad370654f888245f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/66673ea4dfce450abc11133615797c21, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/8a11d4a004434a2cb1c46cacde8c0f1a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/2530c9c7103341919f8ce34670947052, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/6fdf7d58f7fa4a8c879a9d50ce10a68d, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/d48078767d324226b291b3d8bd4730bd, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/f4ba49f743dd4246829e42e38c10970f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/066bc24caf2b4d5980f20c3227be0b97, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/eea796ba5d0a4cd8b0c3a2d7e87ee883, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/100b45950d144ed9b004bcf1fed2f5a3, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/34e6b5fb83934bc29ffa758c80a63fbe, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/07876dfe376449859c1df6b6e9c39085, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/e40a034e04714cae88c0cb965c552b5e, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/c2c1cc9ac66e46b295a64feac4936aa9, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/68682dddab7d4123bc8388281e3a4f05, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/d6948e9d6d604d4caa8dffbcae1a3b85, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/765febf546444668892069630b9edb34, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/3d45999ff82a47b5b500353ddde1fb68, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/ab57f15d225845288bc7ce4ad2bbb169, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/d53ae1a5679b4fd2a34d33e5405bf8df, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/397967ce0d7c4c07ad6ebecd04eedfc8] to archive 2024-11-25T05:57:29,850 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T05:57:29,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/4ff6f500204c4769bcddc48c412a3115 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/4ff6f500204c4769bcddc48c412a3115 2024-11-25T05:57:29,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/d866dfc1e036418dad370654f888245f to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/d866dfc1e036418dad370654f888245f 2024-11-25T05:57:29,854 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/66673ea4dfce450abc11133615797c21 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/66673ea4dfce450abc11133615797c21 2024-11-25T05:57:29,856 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/8a11d4a004434a2cb1c46cacde8c0f1a to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/8a11d4a004434a2cb1c46cacde8c0f1a 2024-11-25T05:57:29,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/2530c9c7103341919f8ce34670947052 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/2530c9c7103341919f8ce34670947052 2024-11-25T05:57:29,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/6fdf7d58f7fa4a8c879a9d50ce10a68d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/6fdf7d58f7fa4a8c879a9d50ce10a68d 2024-11-25T05:57:29,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/d48078767d324226b291b3d8bd4730bd to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/d48078767d324226b291b3d8bd4730bd 2024-11-25T05:57:29,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/f4ba49f743dd4246829e42e38c10970f to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/f4ba49f743dd4246829e42e38c10970f 2024-11-25T05:57:29,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/066bc24caf2b4d5980f20c3227be0b97 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/066bc24caf2b4d5980f20c3227be0b97 2024-11-25T05:57:29,864 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/eea796ba5d0a4cd8b0c3a2d7e87ee883 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/eea796ba5d0a4cd8b0c3a2d7e87ee883 2024-11-25T05:57:29,865 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/100b45950d144ed9b004bcf1fed2f5a3 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/100b45950d144ed9b004bcf1fed2f5a3 2024-11-25T05:57:29,866 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/34e6b5fb83934bc29ffa758c80a63fbe to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/34e6b5fb83934bc29ffa758c80a63fbe 2024-11-25T05:57:29,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/07876dfe376449859c1df6b6e9c39085 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/07876dfe376449859c1df6b6e9c39085 2024-11-25T05:57:29,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/e40a034e04714cae88c0cb965c552b5e to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/e40a034e04714cae88c0cb965c552b5e 2024-11-25T05:57:29,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/c2c1cc9ac66e46b295a64feac4936aa9 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/c2c1cc9ac66e46b295a64feac4936aa9 2024-11-25T05:57:29,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/68682dddab7d4123bc8388281e3a4f05 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/68682dddab7d4123bc8388281e3a4f05 2024-11-25T05:57:29,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/d6948e9d6d604d4caa8dffbcae1a3b85 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/d6948e9d6d604d4caa8dffbcae1a3b85 2024-11-25T05:57:29,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/765febf546444668892069630b9edb34 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/765febf546444668892069630b9edb34 2024-11-25T05:57:29,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/3d45999ff82a47b5b500353ddde1fb68 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/3d45999ff82a47b5b500353ddde1fb68 2024-11-25T05:57:29,875 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/ab57f15d225845288bc7ce4ad2bbb169 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/ab57f15d225845288bc7ce4ad2bbb169 2024-11-25T05:57:29,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/d53ae1a5679b4fd2a34d33e5405bf8df to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/d53ae1a5679b4fd2a34d33e5405bf8df 2024-11-25T05:57:29,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/397967ce0d7c4c07ad6ebecd04eedfc8 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/397967ce0d7c4c07ad6ebecd04eedfc8 2024-11-25T05:57:29,882 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/recovered.edits/318.seqid, newMaxSeqId=318, maxSeqId=1 2024-11-25T05:57:29,883 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. 2024-11-25T05:57:29,883 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1676): Region close journal for 39bebe481dd46e37e6dae001fcad1e53: Waiting for close lock at 1732514248512Running coprocessor pre-close hooks at 1732514248512Disabling compacts and flushes for region at 1732514248512Disabling writes for close at 1732514248512Obtaining lock to block concurrent updates at 1732514248512Preparing flush snapshotting stores in 39bebe481dd46e37e6dae001fcad1e53 at 1732514248512Finished memstore snapshotting TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732514248513 (+1 ms)Flushing stores of TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53. at 1732514248514 (+1 ms)Flushing 39bebe481dd46e37e6dae001fcad1e53/A: creating writer at 1732514248514Flushing 39bebe481dd46e37e6dae001fcad1e53/A: appending metadata at 1732514248518 (+4 ms)Flushing 39bebe481dd46e37e6dae001fcad1e53/A: closing flushed file at 1732514248518Flushing 39bebe481dd46e37e6dae001fcad1e53/B: creating writer at 1732514248933 (+415 ms)Flushing 39bebe481dd46e37e6dae001fcad1e53/B: appending metadata at 1732514248936 (+3 ms)Flushing 39bebe481dd46e37e6dae001fcad1e53/B: closing flushed file at 1732514248936Flushing 39bebe481dd46e37e6dae001fcad1e53/C: creating writer at 1732514249347 (+411 ms)Flushing 39bebe481dd46e37e6dae001fcad1e53/C: appending metadata at 1732514249351 (+4 ms)Flushing 39bebe481dd46e37e6dae001fcad1e53/C: closing flushed file at 1732514249351Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d822e9b: reopening flushed file at 1732514249765 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@268c8c: reopening flushed file at 1732514249772 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c96ddea: reopening flushed file at 1732514249777 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 39bebe481dd46e37e6dae001fcad1e53 in 1270ms, sequenceid=315, compaction requested=true at 1732514249782 (+5 ms)Writing region close event to WAL at 1732514249879 (+97 ms)Running coprocessor post-close hooks at 1732514249883 (+4 ms)Closed at 1732514249883 2024-11-25T05:57:29,885 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] handler.UnassignRegionHandler(157): Closed 39bebe481dd46e37e6dae001fcad1e53 2024-11-25T05:57:29,886 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=127 updating hbase:meta row=39bebe481dd46e37e6dae001fcad1e53, regionState=CLOSED 2024-11-25T05:57:29,887 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=128, ppid=127, state=RUNNABLE, hasLock=false; CloseRegionProcedure 39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 because future has completed 2024-11-25T05:57:29,890 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-25T05:57:29,890 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; CloseRegionProcedure 39bebe481dd46e37e6dae001fcad1e53, server=8ef925b832e3,35599,1732514089005 in 1.5290 sec 2024-11-25T05:57:29,892 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=127, resume processing ppid=126 2024-11-25T05:57:29,892 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, ppid=126, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=39bebe481dd46e37e6dae001fcad1e53, UNASSIGN in 1.5340 sec 2024-11-25T05:57:29,895 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-25T05:57:29,895 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5380 sec 2024-11-25T05:57:29,896 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732514249896"}]},"ts":"1732514249896"} 2024-11-25T05:57:29,898 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-25T05:57:29,898 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-25T05:57:29,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5500 sec 2024-11-25T05:57:30,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-25T05:57:30,502 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:57:30,502 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-25T05:57:30,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T05:57:30,504 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=129, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T05:57:30,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-25T05:57:30,504 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=129, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T05:57:30,506 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53 2024-11-25T05:57:30,508 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A, FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B, FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C, FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/recovered.edits] 2024-11-25T05:57:30,511 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/02b3361d085648e09d067fe98dc56a1e to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/02b3361d085648e09d067fe98dc56a1e 2024-11-25T05:57:30,512 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/26e6714720dc4839affe92848e04a2d1 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/26e6714720dc4839affe92848e04a2d1 2024-11-25T05:57:30,513 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/66b98b87c83c4e58b56b38805ee1886b to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/A/66b98b87c83c4e58b56b38805ee1886b 2024-11-25T05:57:30,516 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/43041955311542da8b7e5a2e7044884c to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/43041955311542da8b7e5a2e7044884c 2024-11-25T05:57:30,517 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/a8e4db9beb5f4b2e9dd57437b72768f9 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/a8e4db9beb5f4b2e9dd57437b72768f9 2024-11-25T05:57:30,518 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/c4a4ebb0b0ca4524ae728e6ec405c35c to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/B/c4a4ebb0b0ca4524ae728e6ec405c35c 2024-11-25T05:57:30,520 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/03eedc3a8d654633bf50bc0be4692fd5 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/03eedc3a8d654633bf50bc0be4692fd5 2024-11-25T05:57:30,521 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/817064493e58452fb01a8e825be1d2eb to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/817064493e58452fb01a8e825be1d2eb 2024-11-25T05:57:30,522 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/f1c6f733969b4401ba42c915f97bbeb0 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/C/f1c6f733969b4401ba42c915f97bbeb0 2024-11-25T05:57:30,524 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/recovered.edits/318.seqid to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53/recovered.edits/318.seqid 2024-11-25T05:57:30,525 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/39bebe481dd46e37e6dae001fcad1e53 2024-11-25T05:57:30,525 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-25T05:57:30,528 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=129, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T05:57:30,530 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-25T05:57:30,533 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-25T05:57:30,534 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=129, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T05:57:30,534 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-25T05:57:30,534 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732514250534"}]},"ts":"9223372036854775807"} 2024-11-25T05:57:30,536 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-25T05:57:30,536 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 39bebe481dd46e37e6dae001fcad1e53, NAME => 'TestAcidGuarantees,,1732514217739.39bebe481dd46e37e6dae001fcad1e53.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T05:57:30,536 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-25T05:57:30,537 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732514250536"}]},"ts":"9223372036854775807"} 2024-11-25T05:57:30,538 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-25T05:57:30,539 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=129, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T05:57:30,540 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 37 msec 2024-11-25T05:57:30,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-25T05:57:30,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-25T05:57:30,618 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:57:30,628 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=246 (was 247), OpenFileDescriptor=441 (was 443), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=293 (was 364), ProcessCount=11 (was 11), AvailableMemoryMB=7589 (was 7611) 2024-11-25T05:57:30,637 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=246, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=293, ProcessCount=11, AvailableMemoryMB=7589 2024-11-25T05:57:30,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T05:57:30,639 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T05:57:30,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=130, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-25T05:57:30,642 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T05:57:30,642 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:57:30,642 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 130 2024-11-25T05:57:30,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-25T05:57:30,643 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T05:57:30,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742347_1523 (size=1061) 2024-11-25T05:57:30,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-25T05:57:30,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-25T05:57:31,055 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a 2024-11-25T05:57:31,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742348_1524 (size=53) 2024-11-25T05:57:31,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-25T05:57:31,467 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T05:57:31,468 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d756b79bb1dbffd6683a6ced0694b917, disabling compactions & flushes 2024-11-25T05:57:31,468 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:31,468 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:31,468 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. after waiting 0 ms 2024-11-25T05:57:31,468 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:31,468 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:31,468 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d756b79bb1dbffd6683a6ced0694b917: Waiting for close lock at 1732514251468Disabling compacts and flushes for region at 1732514251468Disabling writes for close at 1732514251468Writing region close event to WAL at 1732514251468Closed at 1732514251468 2024-11-25T05:57:31,471 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T05:57:31,471 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732514251471"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732514251471"}]},"ts":"1732514251471"} 2024-11-25T05:57:31,474 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T05:57:31,476 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T05:57:31,476 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732514251476"}]},"ts":"1732514251476"} 2024-11-25T05:57:31,478 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-25T05:57:31,479 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d756b79bb1dbffd6683a6ced0694b917, ASSIGN}] 2024-11-25T05:57:31,481 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d756b79bb1dbffd6683a6ced0694b917, ASSIGN 2024-11-25T05:57:31,482 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d756b79bb1dbffd6683a6ced0694b917, ASSIGN; state=OFFLINE, location=8ef925b832e3,35599,1732514089005; forceNewPlan=false, retain=false 2024-11-25T05:57:31,633 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=d756b79bb1dbffd6683a6ced0694b917, regionState=OPENING, regionLocation=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:31,637 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d756b79bb1dbffd6683a6ced0694b917, ASSIGN because future has completed 2024-11-25T05:57:31,638 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; OpenRegionProcedure d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005}] 2024-11-25T05:57:31,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-25T05:57:31,801 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:31,801 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7752): Opening region: {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} 2024-11-25T05:57:31,801 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:31,801 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T05:57:31,801 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7794): checking encryption for d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:31,801 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7797): checking classloading for d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:31,802 INFO [StoreOpener-d756b79bb1dbffd6683a6ced0694b917-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:31,803 INFO [StoreOpener-d756b79bb1dbffd6683a6ced0694b917-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T05:57:31,803 INFO [StoreOpener-d756b79bb1dbffd6683a6ced0694b917-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d756b79bb1dbffd6683a6ced0694b917 columnFamilyName A 2024-11-25T05:57:31,803 DEBUG [StoreOpener-d756b79bb1dbffd6683a6ced0694b917-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:57:31,804 INFO [StoreOpener-d756b79bb1dbffd6683a6ced0694b917-1 {}] regionserver.HStore(327): Store=d756b79bb1dbffd6683a6ced0694b917/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T05:57:31,804 INFO [StoreOpener-d756b79bb1dbffd6683a6ced0694b917-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:31,805 INFO [StoreOpener-d756b79bb1dbffd6683a6ced0694b917-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T05:57:31,805 INFO [StoreOpener-d756b79bb1dbffd6683a6ced0694b917-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d756b79bb1dbffd6683a6ced0694b917 columnFamilyName B 2024-11-25T05:57:31,805 DEBUG [StoreOpener-d756b79bb1dbffd6683a6ced0694b917-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:57:31,805 INFO [StoreOpener-d756b79bb1dbffd6683a6ced0694b917-1 {}] regionserver.HStore(327): Store=d756b79bb1dbffd6683a6ced0694b917/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T05:57:31,805 INFO [StoreOpener-d756b79bb1dbffd6683a6ced0694b917-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:31,806 INFO [StoreOpener-d756b79bb1dbffd6683a6ced0694b917-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T05:57:31,806 INFO [StoreOpener-d756b79bb1dbffd6683a6ced0694b917-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d756b79bb1dbffd6683a6ced0694b917 columnFamilyName C 2024-11-25T05:57:31,806 DEBUG [StoreOpener-d756b79bb1dbffd6683a6ced0694b917-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:57:31,807 INFO [StoreOpener-d756b79bb1dbffd6683a6ced0694b917-1 {}] regionserver.HStore(327): Store=d756b79bb1dbffd6683a6ced0694b917/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T05:57:31,807 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:31,807 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1038): replaying wal for d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:31,807 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:31,808 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:31,808 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1048): stopping wal replay for d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:31,808 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1060): Cleaning up temporary data for d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:31,809 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T05:57:31,809 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1093): writing seq id for d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:31,811 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T05:57:31,811 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1114): Opened d756b79bb1dbffd6683a6ced0694b917; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60652102, jitterRate=-0.09621325135231018}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T05:57:31,811 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:31,812 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1006): Region open journal for d756b79bb1dbffd6683a6ced0694b917: Running coprocessor pre-open hook at 1732514251801Writing region info on filesystem at 1732514251801Initializing all the Stores at 1732514251802 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732514251802Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732514251802Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732514251802Cleaning up temporary data from old regions at 1732514251808 (+6 ms)Running coprocessor post-open hooks at 1732514251811 (+3 ms)Region opened successfully at 1732514251812 (+1 ms) 2024-11-25T05:57:31,812 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., pid=132, masterSystemTime=1732514251793 2024-11-25T05:57:31,814 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:31,814 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:31,815 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=d756b79bb1dbffd6683a6ced0694b917, regionState=OPEN, openSeqNum=2, regionLocation=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:31,816 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=132, ppid=131, state=RUNNABLE, hasLock=false; OpenRegionProcedure d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 because future has completed 2024-11-25T05:57:31,818 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-25T05:57:31,818 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; OpenRegionProcedure d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 in 179 msec 2024-11-25T05:57:31,820 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=131, resume processing ppid=130 2024-11-25T05:57:31,820 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, ppid=130, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d756b79bb1dbffd6683a6ced0694b917, ASSIGN in 339 msec 2024-11-25T05:57:31,821 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T05:57:31,821 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732514251821"}]},"ts":"1732514251821"} 2024-11-25T05:57:31,822 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-25T05:57:31,823 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T05:57:31,824 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1840 sec 2024-11-25T05:57:32,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-25T05:57:32,793 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:57:32,795 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1832522d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:57:32,796 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:57:32,796 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:57:32,798 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:57:32,799 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:57:32,799 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:57:32,799 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e89096c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:57:32,799 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:57:32,800 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:57:32,800 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:32,802 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:57:32,802 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26cefa63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:57:32,803 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:57:32,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:57:32,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:57:32,805 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57908, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:57:32,806 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T05:57:32,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 8ef925b832e3,41285,1732514088402 2024-11-25T05:57:32,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6e0c5838 2024-11-25T05:57:32,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T05:57:32,808 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52802, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T05:57:32,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T05:57:32,810 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T05:57:32,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-25T05:57:32,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742349_1525 (size=1097) 2024-11-25T05:57:33,226 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-25T05:57:33,226 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-25T05:57:33,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T05:57:33,234 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d756b79bb1dbffd6683a6ced0694b917, REOPEN/MOVE}] 2024-11-25T05:57:33,235 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d756b79bb1dbffd6683a6ced0694b917, REOPEN/MOVE 2024-11-25T05:57:33,236 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=d756b79bb1dbffd6683a6ced0694b917, regionState=CLOSING, regionLocation=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:33,237 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d756b79bb1dbffd6683a6ced0694b917, REOPEN/MOVE because future has completed 2024-11-25T05:57:33,237 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T05:57:33,237 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005}] 2024-11-25T05:57:33,391 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(122): Close d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:33,391 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T05:57:33,391 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1722): Closing d756b79bb1dbffd6683a6ced0694b917, disabling compactions & flushes 2024-11-25T05:57:33,391 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:33,391 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:33,391 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. after waiting 0 ms 2024-11-25T05:57:33,391 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:33,400 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T05:57:33,401 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:33,401 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1676): Region close journal for d756b79bb1dbffd6683a6ced0694b917: Waiting for close lock at 1732514253391Running coprocessor pre-close hooks at 1732514253391Disabling compacts and flushes for region at 1732514253391Disabling writes for close at 1732514253391Writing region close event to WAL at 1732514253392 (+1 ms)Running coprocessor post-close hooks at 1732514253401 (+9 ms)Closed at 1732514253401 2024-11-25T05:57:33,401 WARN [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionServer(3299): Not adding moved region record: d756b79bb1dbffd6683a6ced0694b917 to self. 2024-11-25T05:57:33,404 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(157): Closed d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:33,405 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=d756b79bb1dbffd6683a6ced0694b917, regionState=CLOSED 2024-11-25T05:57:33,407 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 because future has completed 2024-11-25T05:57:33,410 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-25T05:57:33,410 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseRegionProcedure d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 in 171 msec 2024-11-25T05:57:33,411 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d756b79bb1dbffd6683a6ced0694b917, REOPEN/MOVE; state=CLOSED, location=8ef925b832e3,35599,1732514089005; forceNewPlan=false, retain=true 2024-11-25T05:57:33,562 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=d756b79bb1dbffd6683a6ced0694b917, regionState=OPENING, regionLocation=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:33,567 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d756b79bb1dbffd6683a6ced0694b917, REOPEN/MOVE because future has completed 2024-11-25T05:57:33,567 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005}] 2024-11-25T05:57:33,729 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:33,729 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(7752): Opening region: {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} 2024-11-25T05:57:33,730 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:33,730 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T05:57:33,730 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(7794): checking encryption for d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:33,731 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(7797): checking classloading for d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:33,732 INFO [StoreOpener-d756b79bb1dbffd6683a6ced0694b917-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:33,732 INFO [StoreOpener-d756b79bb1dbffd6683a6ced0694b917-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T05:57:33,733 INFO [StoreOpener-d756b79bb1dbffd6683a6ced0694b917-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d756b79bb1dbffd6683a6ced0694b917 columnFamilyName A 2024-11-25T05:57:33,734 DEBUG [StoreOpener-d756b79bb1dbffd6683a6ced0694b917-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:57:33,735 INFO [StoreOpener-d756b79bb1dbffd6683a6ced0694b917-1 {}] regionserver.HStore(327): Store=d756b79bb1dbffd6683a6ced0694b917/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T05:57:33,735 INFO [StoreOpener-d756b79bb1dbffd6683a6ced0694b917-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:33,736 INFO [StoreOpener-d756b79bb1dbffd6683a6ced0694b917-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T05:57:33,736 INFO [StoreOpener-d756b79bb1dbffd6683a6ced0694b917-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d756b79bb1dbffd6683a6ced0694b917 columnFamilyName B 2024-11-25T05:57:33,736 DEBUG [StoreOpener-d756b79bb1dbffd6683a6ced0694b917-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:57:33,736 INFO [StoreOpener-d756b79bb1dbffd6683a6ced0694b917-1 {}] regionserver.HStore(327): Store=d756b79bb1dbffd6683a6ced0694b917/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T05:57:33,737 INFO [StoreOpener-d756b79bb1dbffd6683a6ced0694b917-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:33,737 INFO [StoreOpener-d756b79bb1dbffd6683a6ced0694b917-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-25T05:57:33,737 INFO [StoreOpener-d756b79bb1dbffd6683a6ced0694b917-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d756b79bb1dbffd6683a6ced0694b917 columnFamilyName C 2024-11-25T05:57:33,737 DEBUG [StoreOpener-d756b79bb1dbffd6683a6ced0694b917-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:57:33,738 INFO [StoreOpener-d756b79bb1dbffd6683a6ced0694b917-1 {}] regionserver.HStore(327): Store=d756b79bb1dbffd6683a6ced0694b917/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T05:57:33,738 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:33,738 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1038): replaying wal for d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:33,739 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:33,740 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:33,741 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1048): stopping wal replay for d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:33,741 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1060): Cleaning up temporary data for d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:33,742 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T05:57:33,743 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1093): writing seq id for d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:33,744 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1114): Opened d756b79bb1dbffd6683a6ced0694b917; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68888937, jitterRate=0.02652515470981598}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T05:57:33,744 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:33,745 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1006): Region open journal for d756b79bb1dbffd6683a6ced0694b917: Running coprocessor pre-open hook at 1732514253731Writing region info on filesystem at 1732514253731Initializing all the Stores at 1732514253731Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732514253731Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732514253732 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732514253732Cleaning up temporary data from old regions at 1732514253741 (+9 ms)Running coprocessor post-open hooks at 1732514253744 (+3 ms)Region opened successfully at 1732514253745 (+1 ms) 2024-11-25T05:57:33,746 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., pid=137, masterSystemTime=1732514253722 2024-11-25T05:57:33,747 DEBUG [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:33,747 INFO [RS_OPEN_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:33,747 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=d756b79bb1dbffd6683a6ced0694b917, regionState=OPEN, openSeqNum=5, regionLocation=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:33,748 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 because future has completed 2024-11-25T05:57:33,750 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=135 2024-11-25T05:57:33,750 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=135, state=SUCCESS, hasLock=false; OpenRegionProcedure d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 in 182 msec 2024-11-25T05:57:33,752 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-11-25T05:57:33,752 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d756b79bb1dbffd6683a6ced0694b917, REOPEN/MOVE in 517 msec 2024-11-25T05:57:33,754 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-25T05:57:33,754 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 520 msec 2024-11-25T05:57:33,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 943 msec 2024-11-25T05:57:33,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-25T05:57:33,756 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:57:33,756 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22faf5ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:57:33,756 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:57:33,757 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:57:33,757 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:57:33,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:57:33,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:57:33,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2382687d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:57:33,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:57:33,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:57:33,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:33,758 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:57:33,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49385595, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:57:33,759 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@744b0f71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:57:33,759 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:57:33,759 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:57:33,760 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:57:33,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:57:33,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:57:33,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4283eae5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:57:33,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:57:33,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:57:33,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:33,761 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52846, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:57:33,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af567e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:57:33,762 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143fb08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:57:33,762 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:57:33,762 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:57:33,762 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:57:33,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:57:33,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:57:33,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@705a5c6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:57:33,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:57:33,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:57:33,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:33,763 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52860, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:57:33,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a91cda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:57:33,764 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40503a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:57:33,764 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:57:33,764 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:57:33,765 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:57:33,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:57:33,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:57:33,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d565bc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:57:33,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:57:33,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:57:33,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:33,766 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52876, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:57:33,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@757b6273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:57:33,766 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f72e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:57:33,766 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:57:33,767 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:57:33,767 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:57:33,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:57:33,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:57:33,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@213625d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:57:33,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:57:33,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:57:33,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:33,768 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52902, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:57:33,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ea0c8a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:57:33,769 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763ca8cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:57:33,769 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:57:33,769 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:57:33,770 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:57:33,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:57:33,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:57:33,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b54c93f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:57:33,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:57:33,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:57:33,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:33,771 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52918, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:57:33,771 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d29e4cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:57:33,772 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7122db2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:57:33,772 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:57:33,772 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:57:33,773 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:57:33,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:57:33,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:57:33,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d693487, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:57:33,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:57:33,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:57:33,774 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:33,774 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52924, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:57:33,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51b3d0f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:57:33,775 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3666b173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:57:33,775 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:57:33,775 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:57:33,776 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:57:33,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:57:33,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:57:33,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@280e03a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:57:33,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:57:33,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:57:33,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:33,778 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52938, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:57:33,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c383682, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:57:33,779 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ff027c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:57:33,779 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:57:33,779 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:57:33,780 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:57:33,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:57:33,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:57:33,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68c19224, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:57:33,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:57:33,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:57:33,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:33,781 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52958, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:57:33,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a7e3ec8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:57:33,782 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2637386d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:57:33,782 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8ef925b832e3,41285,-1 for getting cluster id 2024-11-25T05:57:33,782 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T05:57:33,782 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0e308606-44a7-4b56-9484-ef3e65c21c05' 2024-11-25T05:57:33,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T05:57:33,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0e308606-44a7-4b56-9484-ef3e65c21c05" 2024-11-25T05:57:33,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a9c32f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:57:33,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8ef925b832e3,41285,-1] 2024-11-25T05:57:33,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T05:57:33,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:33,783 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52972, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T05:57:33,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65b6919e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T05:57:33,784 DEBUG [Thread-2348 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:57:33,784 DEBUG [Thread-2365 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:57:33,785 DEBUG [Thread-2363 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:57:33,785 DEBUG [Thread-2352 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:57:33,785 DEBUG [Thread-2346 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:57:33,785 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:57:33,785 DEBUG [Thread-2357 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:57:33,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:57:33,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:57:33,785 DEBUG [Thread-2359 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:57:33,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:57:33,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:57:33,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:57:33,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:57:33,785 DEBUG [Thread-2354 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:57:33,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:57:33,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:57:33,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:57:33,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees 2024-11-25T05:57:33,786 DEBUG [Thread-2350 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:57:33,786 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:57:33,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:57:33,786 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:57:33,787 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:57:33,787 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57946, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:57:33,787 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:57:33,787 DEBUG [Thread-2361 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T05:57:33,787 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57972, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:57:33,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-25T05:57:33,787 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:57:33,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:57:33,787 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:57:33,787 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57962, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:57:33,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:57:33,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:57:33,788 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57920, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:57:33,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:57:33,788 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57984, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:57:33,788 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57988, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:57:33,788 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:57:33,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8ef925b832e3,35599,1732514089005, seqNum=-1] 2024-11-25T05:57:33,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5] 2024-11-25T05:57:33,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T05:57:33,789 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=138, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:57:33,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5] 2024-11-25T05:57:33,790 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58000, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T05:57:33,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5] 2024-11-25T05:57:33,790 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=138, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:57:33,790 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:57:33,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5] 2024-11-25T05:57:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5] 2024-11-25T05:57:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5] 2024-11-25T05:57:33,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5] 2024-11-25T05:57:33,792 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5] 2024-11-25T05:57:33,792 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5] 2024-11-25T05:57:33,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5] 2024-11-25T05:57:33,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:33,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d756b79bb1dbffd6683a6ced0694b917 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-25T05:57:33,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=A 2024-11-25T05:57:33,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:33,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=B 2024-11-25T05:57:33,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:33,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=C 2024-11-25T05:57:33,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:33,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:33,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514313811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:33,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:33,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:33,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:33,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732514253807, max timestamp=1732514253814Num uniques 859; 2024-11-25T05:57:33,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T05:57:33,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732514253807, max timestamp=1732514253814Num uniques 859; 2024-11-25T05:57:33,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T05:57:33,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732514253807, max timestamp=1732514253814Num uniques 859; 2024-11-25T05:57:33,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8ef925b832e3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-25T05:57:33,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112547bac1310c0842d1ba1f6f1fc2ca2398_d756b79bb1dbffd6683a6ced0694b917 is 50, key is test_row_1/A:col10/1732514253796/Put/seqid=0 2024-11-25T05:57:33,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742350_1526 (size=9714) 2024-11-25T05:57:33,827 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:57:33,830 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112547bac1310c0842d1ba1f6f1fc2ca2398_d756b79bb1dbffd6683a6ced0694b917 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112547bac1310c0842d1ba1f6f1fc2ca2398_d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:33,831 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/55e9f73e23f14d04aeceba53191115b4, store: [table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:33,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/55e9f73e23f14d04aeceba53191115b4 is 175, key is test_row_1/A:col10/1732514253796/Put/seqid=0 2024-11-25T05:57:33,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742351_1527 (size=22361) 2024-11-25T05:57:33,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:33,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:33,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514313833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:33,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514313833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:33,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:33,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:33,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:33,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:33,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:33,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:33,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:33,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57984 deadline: 1732514313837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:33,841 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:33,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:33,841 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:33,841 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:33,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1732514313837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:33,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:33,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:33,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:33,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-25T05:57:33,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:33,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514313921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:33,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:33,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:33,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:33,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-25T05:57:33,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:33,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. as already flushing 2024-11-25T05:57:33,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:33,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:33,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:33,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:33,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:33,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514313942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:33,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514313942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:33,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:33,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:33,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:33,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:33,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:33,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:33,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:33,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:33,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57984 deadline: 1732514313952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:33,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:33,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:33,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1732514313952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:33,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:33,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:33,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:33,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:33,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:34,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-25T05:57:34,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:34,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. as already flushing 2024-11-25T05:57:34,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:34,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:34,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:34,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:34,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-25T05:57:34,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:34,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514314132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:34,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:34,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:34,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:34,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:34,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:34,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514314152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:34,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514314152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:34,156 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:34,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:34,156 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:34,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:34,156 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:34,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:34,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:34,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57984 deadline: 1732514314161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:34,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:34,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:34,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:34,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:34,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1732514314161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:34,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:34,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:34,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:34,235 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/55e9f73e23f14d04aeceba53191115b4 2024-11-25T05:57:34,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-25T05:57:34,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:34,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. as already flushing 2024-11-25T05:57:34,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:34,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:34,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:34,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:34,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/a0a15d55e1dd48c9b9542291a0619095 is 50, key is test_row_1/B:col10/1732514253796/Put/seqid=0 2024-11-25T05:57:34,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742352_1528 (size=9657) 2024-11-25T05:57:34,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-25T05:57:34,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:34,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. as already flushing 2024-11-25T05:57:34,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:34,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:34,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:34,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:34,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-25T05:57:34,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:34,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514314442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:34,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:34,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:34,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:34,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:34,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514314462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:34,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:34,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:34,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514314462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:34,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:34,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:34,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:34,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:34,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:34,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:34,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57984 deadline: 1732514314472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:34,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:34,476 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:34,476 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:34,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1732514314472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:34,476 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:34,476 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:34,476 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:34,476 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:34,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-25T05:57:34,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:34,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. as already flushing 2024-11-25T05:57:34,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:34,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:34,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:34,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:34,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/a0a15d55e1dd48c9b9542291a0619095 2024-11-25T05:57:34,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/5c54f273e3494abfb92133b699696006 is 50, key is test_row_1/C:col10/1732514253796/Put/seqid=0 2024-11-25T05:57:34,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742353_1529 (size=9657) 2024-11-25T05:57:34,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/5c54f273e3494abfb92133b699696006 2024-11-25T05:57:34,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/55e9f73e23f14d04aeceba53191115b4 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/55e9f73e23f14d04aeceba53191115b4 2024-11-25T05:57:34,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/55e9f73e23f14d04aeceba53191115b4, entries=100, sequenceid=15, filesize=21.8 K 2024-11-25T05:57:34,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/a0a15d55e1dd48c9b9542291a0619095 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/a0a15d55e1dd48c9b9542291a0619095 2024-11-25T05:57:34,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/a0a15d55e1dd48c9b9542291a0619095, entries=100, sequenceid=15, filesize=9.4 K 2024-11-25T05:57:34,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/5c54f273e3494abfb92133b699696006 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/5c54f273e3494abfb92133b699696006 2024-11-25T05:57:34,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/5c54f273e3494abfb92133b699696006, entries=100, sequenceid=15, filesize=9.4 K 2024-11-25T05:57:34,699 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for d756b79bb1dbffd6683a6ced0694b917 in 903ms, sequenceid=15, compaction requested=false 2024-11-25T05:57:34,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d756b79bb1dbffd6683a6ced0694b917: 2024-11-25T05:57:34,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-25T05:57:34,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:34,706 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2902): Flushing d756b79bb1dbffd6683a6ced0694b917 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-25T05:57:34,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=A 2024-11-25T05:57:34,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:57:34,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=B 2024-11-25T05:57:34,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:57:34,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=C 2024-11-25T05:57:34,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-25T05:57:34,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112539a9a88387fe405f82439c1ec6cab177_d756b79bb1dbffd6683a6ced0694b917 is 50, key is test_row_0/A:col10/1732514253836/Put/seqid=0 2024-11-25T05:57:34,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742354_1530 (size=12154) 2024-11-25T05:57:34,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-25T05:57:34,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:34,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. as already flushing 2024-11-25T05:57:34,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:34,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514314975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:34,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:34,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:34,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:34,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514314976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:34,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514314977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:34,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:34,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57984 deadline: 1732514314982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:34,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:34,983 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:34,983 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:34,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1732514314982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:34,983 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:34,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:34,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:34,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:35,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:35,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514315082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:35,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:35,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:35,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:35,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:35,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514315092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:35,094 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:35,094 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:35,094 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:35,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:35,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514315092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:35,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:35,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:35,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:35,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:57:35,131 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112539a9a88387fe405f82439c1ec6cab177_d756b79bb1dbffd6683a6ced0694b917 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112539a9a88387fe405f82439c1ec6cab177_d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:35,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/6af42a61390d47f88235c05722d79469, store: [table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:35,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/6af42a61390d47f88235c05722d79469 is 175, key is test_row_0/A:col10/1732514253836/Put/seqid=0 2024-11-25T05:57:35,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742355_1531 (size=30955) 2024-11-25T05:57:35,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:35,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514315292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:35,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:35,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:35,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:35,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:35,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514315301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:35,305 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:35,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:35,305 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:35,305 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:35,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514315301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:35,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:35,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:35,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:35,400 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T05:57:35,535 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/6af42a61390d47f88235c05722d79469 2024-11-25T05:57:35,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/690c65d9842645c9ab83a045c1e592e3 is 50, key is test_row_0/B:col10/1732514253836/Put/seqid=0 2024-11-25T05:57:35,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742356_1532 (size=12001) 2024-11-25T05:57:35,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:35,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514315602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:35,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:35,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:35,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:35,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:35,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514315612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:35,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:35,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514315612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:35,614 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:35,614 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:35,614 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:35,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:35,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:35,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-25T05:57:35,945 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/690c65d9842645c9ab83a045c1e592e3 2024-11-25T05:57:35,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/e1426950801143e0beb1548d61efdcbb is 50, key is test_row_0/C:col10/1732514253836/Put/seqid=0 2024-11-25T05:57:35,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742357_1533 (size=12001) 2024-11-25T05:57:35,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:35,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1732514315992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:35,996 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_2' in d756b79bb1dbffd6683a6ced0694b917 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:57:35,997 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:35,997 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:35,997 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:36,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:36,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57984 deadline: 1732514316002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:36,005 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_2' in d756b79bb1dbffd6683a6ced0694b917 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:57:36,005 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:36,005 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:36,005 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:36,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514316112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:36,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:36,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:36,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:36,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:36,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514316122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:36,125 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:36,125 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:36,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:36,125 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:36,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514316122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:36,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:36,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:36,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:36,355 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/e1426950801143e0beb1548d61efdcbb 2024-11-25T05:57:36,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/6af42a61390d47f88235c05722d79469 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/6af42a61390d47f88235c05722d79469 2024-11-25T05:57:36,361 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/6af42a61390d47f88235c05722d79469, entries=150, sequenceid=50, filesize=30.2 K 2024-11-25T05:57:36,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/690c65d9842645c9ab83a045c1e592e3 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/690c65d9842645c9ab83a045c1e592e3 2024-11-25T05:57:36,364 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/690c65d9842645c9ab83a045c1e592e3, entries=150, sequenceid=50, filesize=11.7 K 2024-11-25T05:57:36,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/e1426950801143e0beb1548d61efdcbb as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/e1426950801143e0beb1548d61efdcbb 2024-11-25T05:57:36,368 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/e1426950801143e0beb1548d61efdcbb, entries=150, sequenceid=50, filesize=11.7 K 2024-11-25T05:57:36,368 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for d756b79bb1dbffd6683a6ced0694b917 in 1662ms, sequenceid=50, compaction requested=false 2024-11-25T05:57:36,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2603): Flush status journal for d756b79bb1dbffd6683a6ced0694b917: 2024-11-25T05:57:36,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:36,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=139 2024-11-25T05:57:36,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=139 2024-11-25T05:57:36,371 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-25T05:57:36,371 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5790 sec 2024-11-25T05:57:36,372 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees in 2.5860 sec 2024-11-25T05:57:37,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:37,134 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d756b79bb1dbffd6683a6ced0694b917 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-25T05:57:37,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=A 2024-11-25T05:57:37,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:37,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=B 2024-11-25T05:57:37,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:37,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=C 2024-11-25T05:57:37,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:37,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411257c996edb56144cc2a321119099430f36_d756b79bb1dbffd6683a6ced0694b917 is 50, key is test_row_0/A:col10/1732514254974/Put/seqid=0 2024-11-25T05:57:37,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742358_1534 (size=14594) 2024-11-25T05:57:37,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:37,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514317222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:37,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:37,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:37,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:37,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514317223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:37,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:37,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514317223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:37,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:37,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:37,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:37,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:37,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:37,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:37,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514317332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:37,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:37,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514317332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:37,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:37,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:37,335 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:37,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:37,335 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:37,335 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514317332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:37,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:37,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:37,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:37,543 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:57:37,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:37,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514317542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:37,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:37,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:37,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:37,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514317542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:37,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:37,545 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:37,545 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:37,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:37,545 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:37,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514317542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:37,546 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411257c996edb56144cc2a321119099430f36_d756b79bb1dbffd6683a6ced0694b917 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411257c996edb56144cc2a321119099430f36_d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:37,546 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/b2c691d7963747079d353bdf91c35f7a, store: [table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:37,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/b2c691d7963747079d353bdf91c35f7a is 175, key is test_row_0/A:col10/1732514254974/Put/seqid=0 2024-11-25T05:57:37,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:37,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:37,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:37,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742359_1535 (size=39549) 2024-11-25T05:57:37,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:37,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514317852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:37,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:37,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:37,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:37,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:37,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514317852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:37,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:37,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514317853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:37,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-25T05:57:37,943 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:57:37,943 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:57:37,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees 2024-11-25T05:57:37,945 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:57:37,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-25T05:57:37,945 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:57:37,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:57:37,950 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/b2c691d7963747079d353bdf91c35f7a 2024-11-25T05:57:37,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/8a3c4847538a41d389f3bd8ebb35996b is 50, key is test_row_0/B:col10/1732514254974/Put/seqid=0 2024-11-25T05:57:37,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742360_1536 (size=12001) 2024-11-25T05:57:38,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:38,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1732514318022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:38,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:38,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57984 deadline: 1732514318022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:38,025 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_2' in d756b79bb1dbffd6683a6ced0694b917 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:57:38,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:38,025 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_2' in d756b79bb1dbffd6683a6ced0694b917 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:57:38,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:38,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:38,026 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:38,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:38,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:38,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-25T05:57:38,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-25T05:57:38,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:38,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. as already flushing 2024-11-25T05:57:38,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:38,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:38,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:38,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:38,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-25T05:57:38,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:38,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. as already flushing 2024-11-25T05:57:38,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:38,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:38,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:38,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:38,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-25T05:57:38,362 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/8a3c4847538a41d389f3bd8ebb35996b 2024-11-25T05:57:38,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:38,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514318362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:38,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:38,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514318362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:38,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:38,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:38,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:38,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:38,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514318362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:38,365 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:38,365 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:38,365 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:38,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:38,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:38,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:38,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/4deef89ffeef4983a817efd7bccc3e3a is 50, key is test_row_0/C:col10/1732514254974/Put/seqid=0 2024-11-25T05:57:38,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742361_1537 (size=12001) 2024-11-25T05:57:38,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-25T05:57:38,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:38,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. as already flushing 2024-11-25T05:57:38,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:38,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:38,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:38,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:38,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-25T05:57:38,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:38,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. as already flushing 2024-11-25T05:57:38,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:38,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:38,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:38,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:38,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-25T05:57:38,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-25T05:57:38,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:38,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. as already flushing 2024-11-25T05:57:38,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:38,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:38,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:38,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:38,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/4deef89ffeef4983a817efd7bccc3e3a 2024-11-25T05:57:38,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/b2c691d7963747079d353bdf91c35f7a as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/b2c691d7963747079d353bdf91c35f7a 2024-11-25T05:57:38,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/b2c691d7963747079d353bdf91c35f7a, entries=200, sequenceid=62, filesize=38.6 K 2024-11-25T05:57:38,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/8a3c4847538a41d389f3bd8ebb35996b as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/8a3c4847538a41d389f3bd8ebb35996b 2024-11-25T05:57:38,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/8a3c4847538a41d389f3bd8ebb35996b, entries=150, sequenceid=62, filesize=11.7 K 2024-11-25T05:57:38,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/4deef89ffeef4983a817efd7bccc3e3a as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/4deef89ffeef4983a817efd7bccc3e3a 2024-11-25T05:57:38,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/4deef89ffeef4983a817efd7bccc3e3a, entries=150, sequenceid=62, filesize=11.7 K 2024-11-25T05:57:38,797 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for d756b79bb1dbffd6683a6ced0694b917 in 1663ms, sequenceid=62, compaction requested=true 2024-11-25T05:57:38,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d756b79bb1dbffd6683a6ced0694b917: 2024-11-25T05:57:38,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d756b79bb1dbffd6683a6ced0694b917:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:57:38,798 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:57:38,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:38,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d756b79bb1dbffd6683a6ced0694b917:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:57:38,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:38,798 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:57:38,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d756b79bb1dbffd6683a6ced0694b917:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:57:38,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:57:38,798 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:57:38,798 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:57:38,798 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): d756b79bb1dbffd6683a6ced0694b917/A is initiating minor compaction (all files) 2024-11-25T05:57:38,798 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): d756b79bb1dbffd6683a6ced0694b917/B is initiating minor compaction (all files) 2024-11-25T05:57:38,798 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d756b79bb1dbffd6683a6ced0694b917/B in TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:38,798 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d756b79bb1dbffd6683a6ced0694b917/A in TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:38,798 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/55e9f73e23f14d04aeceba53191115b4, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/6af42a61390d47f88235c05722d79469, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/b2c691d7963747079d353bdf91c35f7a] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp, totalSize=90.7 K 2024-11-25T05:57:38,798 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/a0a15d55e1dd48c9b9542291a0619095, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/690c65d9842645c9ab83a045c1e592e3, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/8a3c4847538a41d389f3bd8ebb35996b] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp, totalSize=32.9 K 2024-11-25T05:57:38,798 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:38,799 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. files: [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/55e9f73e23f14d04aeceba53191115b4, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/6af42a61390d47f88235c05722d79469, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/b2c691d7963747079d353bdf91c35f7a] 2024-11-25T05:57:38,799 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting a0a15d55e1dd48c9b9542291a0619095, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732514253793 2024-11-25T05:57:38,799 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55e9f73e23f14d04aeceba53191115b4, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732514253793 2024-11-25T05:57:38,799 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 690c65d9842645c9ab83a045c1e592e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732514253824 2024-11-25T05:57:38,799 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6af42a61390d47f88235c05722d79469, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732514253824 2024-11-25T05:57:38,799 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a3c4847538a41d389f3bd8ebb35996b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732514254974 2024-11-25T05:57:38,799 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2c691d7963747079d353bdf91c35f7a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732514254974 2024-11-25T05:57:38,804 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:38,805 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d756b79bb1dbffd6683a6ced0694b917#B#compaction#461 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:38,806 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/f775ce5056b24c0497cf37bbd9ed9951 is 50, key is test_row_0/B:col10/1732514254974/Put/seqid=0 2024-11-25T05:57:38,806 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125cdaa2978099a462b8f8fffe8178b45d1_d756b79bb1dbffd6683a6ced0694b917 store=[table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:38,808 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125cdaa2978099a462b8f8fffe8178b45d1_d756b79bb1dbffd6683a6ced0694b917, store=[table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:38,809 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125cdaa2978099a462b8f8fffe8178b45d1_d756b79bb1dbffd6683a6ced0694b917 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:38,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742363_1539 (size=4469) 2024-11-25T05:57:38,816 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d756b79bb1dbffd6683a6ced0694b917#A#compaction#462 average throughput is 2.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:38,817 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/2604ea92049942cd8a5d94b665acaf42 is 175, key is test_row_0/A:col10/1732514254974/Put/seqid=0 2024-11-25T05:57:38,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742362_1538 (size=12104) 2024-11-25T05:57:38,823 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/f775ce5056b24c0497cf37bbd9ed9951 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/f775ce5056b24c0497cf37bbd9ed9951 2024-11-25T05:57:38,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742364_1540 (size=31058) 2024-11-25T05:57:38,829 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d756b79bb1dbffd6683a6ced0694b917/B of d756b79bb1dbffd6683a6ced0694b917 into f775ce5056b24c0497cf37bbd9ed9951(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:38,829 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d756b79bb1dbffd6683a6ced0694b917: 2024-11-25T05:57:38,829 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., storeName=d756b79bb1dbffd6683a6ced0694b917/B, priority=13, startTime=1732514258798; duration=0sec 2024-11-25T05:57:38,829 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:57:38,829 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d756b79bb1dbffd6683a6ced0694b917:B 2024-11-25T05:57:38,829 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:57:38,830 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:57:38,830 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): d756b79bb1dbffd6683a6ced0694b917/C is initiating minor compaction (all files) 2024-11-25T05:57:38,830 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d756b79bb1dbffd6683a6ced0694b917/C in TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:38,831 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/5c54f273e3494abfb92133b699696006, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/e1426950801143e0beb1548d61efdcbb, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/4deef89ffeef4983a817efd7bccc3e3a] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp, totalSize=32.9 K 2024-11-25T05:57:38,831 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c54f273e3494abfb92133b699696006, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732514253793 2024-11-25T05:57:38,831 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting e1426950801143e0beb1548d61efdcbb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732514253824 2024-11-25T05:57:38,831 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 4deef89ffeef4983a817efd7bccc3e3a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732514254974 2024-11-25T05:57:38,832 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/2604ea92049942cd8a5d94b665acaf42 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/2604ea92049942cd8a5d94b665acaf42 2024-11-25T05:57:38,839 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d756b79bb1dbffd6683a6ced0694b917/A of d756b79bb1dbffd6683a6ced0694b917 into 2604ea92049942cd8a5d94b665acaf42(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:38,839 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d756b79bb1dbffd6683a6ced0694b917: 2024-11-25T05:57:38,839 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., storeName=d756b79bb1dbffd6683a6ced0694b917/A, priority=13, startTime=1732514258797; duration=0sec 2024-11-25T05:57:38,839 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:38,839 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d756b79bb1dbffd6683a6ced0694b917:A 2024-11-25T05:57:38,839 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d756b79bb1dbffd6683a6ced0694b917#C#compaction#463 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:38,839 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/d8a6709cf3e04140b9c9d8be80405073 is 50, key is test_row_0/C:col10/1732514254974/Put/seqid=0 2024-11-25T05:57:38,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742365_1541 (size=12104) 2024-11-25T05:57:38,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-25T05:57:38,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:38,860 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2902): Flushing d756b79bb1dbffd6683a6ced0694b917 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-25T05:57:38,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=A 2024-11-25T05:57:38,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:38,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=B 2024-11-25T05:57:38,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:38,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=C 2024-11-25T05:57:38,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:38,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112581f29ef5208343129d3abe5fc7a5c683_d756b79bb1dbffd6683a6ced0694b917 is 50, key is test_row_0/A:col10/1732514257221/Put/seqid=0 2024-11-25T05:57:38,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742366_1542 (size=12154) 2024-11-25T05:57:39,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-25T05:57:39,247 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/d8a6709cf3e04140b9c9d8be80405073 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/d8a6709cf3e04140b9c9d8be80405073 2024-11-25T05:57:39,251 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d756b79bb1dbffd6683a6ced0694b917/C of d756b79bb1dbffd6683a6ced0694b917 into d8a6709cf3e04140b9c9d8be80405073(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:39,251 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d756b79bb1dbffd6683a6ced0694b917: 2024-11-25T05:57:39,251 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., storeName=d756b79bb1dbffd6683a6ced0694b917/C, priority=13, startTime=1732514258798; duration=0sec 2024-11-25T05:57:39,251 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:39,251 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d756b79bb1dbffd6683a6ced0694b917:C 2024-11-25T05:57:39,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:57:39,273 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112581f29ef5208343129d3abe5fc7a5c683_d756b79bb1dbffd6683a6ced0694b917 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112581f29ef5208343129d3abe5fc7a5c683_d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:39,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/b227b28b153841f2b7d8e66511037cb3, store: [table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:39,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/b227b28b153841f2b7d8e66511037cb3 is 175, key is test_row_0/A:col10/1732514257221/Put/seqid=0 2024-11-25T05:57:39,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742367_1543 (size=30955) 2024-11-25T05:57:39,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:39,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. as already flushing 2024-11-25T05:57:39,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:39,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514319391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:39,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:39,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:39,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:39,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:39,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514319392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:39,395 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:39,395 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:39,395 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:39,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:39,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514319393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:39,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:39,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:39,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:39,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:39,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514319502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:39,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:39,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:39,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:39,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:39,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:39,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514319502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:39,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514319502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:39,505 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:39,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:39,505 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:39,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:39,505 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:39,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:39,678 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=84, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/b227b28b153841f2b7d8e66511037cb3 2024-11-25T05:57:39,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/c40046aee2974ff3bd14fee1e595b945 is 50, key is test_row_0/B:col10/1732514257221/Put/seqid=0 2024-11-25T05:57:39,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742368_1544 (size=12001) 2024-11-25T05:57:39,688 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/c40046aee2974ff3bd14fee1e595b945 2024-11-25T05:57:39,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/8e9f3bc74bca48709781ea0e4bdb5840 is 50, key is test_row_0/C:col10/1732514257221/Put/seqid=0 2024-11-25T05:57:39,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742369_1545 (size=12001) 2024-11-25T05:57:39,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:39,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514319712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:39,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:39,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:39,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:39,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:39,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514319712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:39,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:39,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514319712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:39,715 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:39,715 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:39,715 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:39,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:39,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:39,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:40,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514320022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:40,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:40,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514320022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:40,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:40,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:40,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:40,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:40,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:40,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514320022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:40,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:40,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:40,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:40,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:40,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-25T05:57:40,099 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/8e9f3bc74bca48709781ea0e4bdb5840 2024-11-25T05:57:40,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/b227b28b153841f2b7d8e66511037cb3 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/b227b28b153841f2b7d8e66511037cb3 2024-11-25T05:57:40,105 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/b227b28b153841f2b7d8e66511037cb3, entries=150, sequenceid=84, filesize=30.2 K 2024-11-25T05:57:40,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/c40046aee2974ff3bd14fee1e595b945 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/c40046aee2974ff3bd14fee1e595b945 2024-11-25T05:57:40,108 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/c40046aee2974ff3bd14fee1e595b945, entries=150, sequenceid=84, filesize=11.7 K 2024-11-25T05:57:40,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/8e9f3bc74bca48709781ea0e4bdb5840 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/8e9f3bc74bca48709781ea0e4bdb5840 2024-11-25T05:57:40,111 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/8e9f3bc74bca48709781ea0e4bdb5840, entries=150, sequenceid=84, filesize=11.7 K 2024-11-25T05:57:40,112 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for d756b79bb1dbffd6683a6ced0694b917 in 1252ms, sequenceid=84, compaction requested=false 2024-11-25T05:57:40,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2603): Flush status journal for d756b79bb1dbffd6683a6ced0694b917: 2024-11-25T05:57:40,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:40,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=141 2024-11-25T05:57:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=141 2024-11-25T05:57:40,114 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-25T05:57:40,114 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1680 sec 2024-11-25T05:57:40,116 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees in 2.1720 sec 2024-11-25T05:57:40,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:40,535 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d756b79bb1dbffd6683a6ced0694b917 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-25T05:57:40,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=A 2024-11-25T05:57:40,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:40,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=B 2024-11-25T05:57:40,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:40,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=C 2024-11-25T05:57:40,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:40,541 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c1d8ce4733ef44fea1ebcf348b3067ea_d756b79bb1dbffd6683a6ced0694b917 is 50, key is test_row_0/A:col10/1732514259390/Put/seqid=0 2024-11-25T05:57:40,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742370_1546 (size=12154) 2024-11-25T05:57:40,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:40,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514320557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:40,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:40,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514320558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:40,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:40,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:40,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:40,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:40,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:40,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:40,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:40,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514320561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:40,565 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:40,565 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:40,565 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:40,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:40,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514320672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:40,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:40,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:40,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:40,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:40,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:40,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514320673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:40,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514320673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:40,676 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:40,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:40,676 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:40,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:40,676 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:40,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:40,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:40,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514320882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:40,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:40,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:40,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:40,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514320882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:40,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:40,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:40,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:40,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:40,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:40,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514320882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:40,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:40,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:40,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:40,944 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:57:40,947 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c1d8ce4733ef44fea1ebcf348b3067ea_d756b79bb1dbffd6683a6ced0694b917 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c1d8ce4733ef44fea1ebcf348b3067ea_d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:40,947 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/c222fdb5749d47cfa45fb351c787ad9c, store: [table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:40,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/c222fdb5749d47cfa45fb351c787ad9c is 175, key is test_row_0/A:col10/1732514259390/Put/seqid=0 2024-11-25T05:57:40,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742371_1547 (size=30955) 2024-11-25T05:57:41,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:41,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514321192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:41,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:41,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:41,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:41,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:41,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514321192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:41,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:41,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:41,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514321192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:41,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:41,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:41,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:41,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:41,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:41,351 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/c222fdb5749d47cfa45fb351c787ad9c 2024-11-25T05:57:41,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/dd0a16b028da43378ef1c338e752124e is 50, key is test_row_0/B:col10/1732514259390/Put/seqid=0 2024-11-25T05:57:41,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742372_1548 (size=12001) 2024-11-25T05:57:41,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:41,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514321702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:41,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:41,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:41,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514321702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:41,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:41,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:41,703 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:41,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:41,703 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:41,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514321702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:41,703 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:41,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:41,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:41,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:41,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/dd0a16b028da43378ef1c338e752124e 2024-11-25T05:57:41,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/ae96ae6839974d42b450eba59ee64697 is 50, key is test_row_0/C:col10/1732514259390/Put/seqid=0 2024-11-25T05:57:41,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742373_1549 (size=12001) 2024-11-25T05:57:42,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:42,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57984 deadline: 1732514322042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:42,044 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_2' in d756b79bb1dbffd6683a6ced0694b917 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:57:42,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:42,045 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:42,045 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:42,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:42,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1732514322052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:42,055 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_2' in d756b79bb1dbffd6683a6ced0694b917 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:57:42,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:42,056 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:42,056 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:42,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-25T05:57:42,112 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:57:42,112 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:57:42,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-11-25T05:57:42,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-25T05:57:42,114 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:57:42,115 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:57:42,115 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:57:42,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/ae96ae6839974d42b450eba59ee64697 2024-11-25T05:57:42,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/c222fdb5749d47cfa45fb351c787ad9c as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/c222fdb5749d47cfa45fb351c787ad9c 2024-11-25T05:57:42,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/c222fdb5749d47cfa45fb351c787ad9c, entries=150, sequenceid=98, filesize=30.2 K 2024-11-25T05:57:42,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/dd0a16b028da43378ef1c338e752124e as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/dd0a16b028da43378ef1c338e752124e 2024-11-25T05:57:42,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/dd0a16b028da43378ef1c338e752124e, entries=150, sequenceid=98, filesize=11.7 K 2024-11-25T05:57:42,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/ae96ae6839974d42b450eba59ee64697 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/ae96ae6839974d42b450eba59ee64697 2024-11-25T05:57:42,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/ae96ae6839974d42b450eba59ee64697, entries=150, sequenceid=98, filesize=11.7 K 2024-11-25T05:57:42,183 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for d756b79bb1dbffd6683a6ced0694b917 in 1648ms, sequenceid=98, compaction requested=true 2024-11-25T05:57:42,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d756b79bb1dbffd6683a6ced0694b917: 2024-11-25T05:57:42,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d756b79bb1dbffd6683a6ced0694b917:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:57:42,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:42,184 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:57:42,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d756b79bb1dbffd6683a6ced0694b917:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:57:42,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:42,184 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:57:42,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d756b79bb1dbffd6683a6ced0694b917:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:57:42,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:57:42,185 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:57:42,185 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): d756b79bb1dbffd6683a6ced0694b917/A is initiating minor compaction (all files) 2024-11-25T05:57:42,185 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d756b79bb1dbffd6683a6ced0694b917/A in TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:42,185 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:57:42,185 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): d756b79bb1dbffd6683a6ced0694b917/B is initiating minor compaction (all files) 2024-11-25T05:57:42,185 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/2604ea92049942cd8a5d94b665acaf42, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/b227b28b153841f2b7d8e66511037cb3, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/c222fdb5749d47cfa45fb351c787ad9c] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp, totalSize=90.8 K 2024-11-25T05:57:42,185 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d756b79bb1dbffd6683a6ced0694b917/B in TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:42,185 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:42,185 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. files: [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/2604ea92049942cd8a5d94b665acaf42, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/b227b28b153841f2b7d8e66511037cb3, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/c222fdb5749d47cfa45fb351c787ad9c] 2024-11-25T05:57:42,185 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/f775ce5056b24c0497cf37bbd9ed9951, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/c40046aee2974ff3bd14fee1e595b945, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/dd0a16b028da43378ef1c338e752124e] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp, totalSize=35.3 K 2024-11-25T05:57:42,185 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2604ea92049942cd8a5d94b665acaf42, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732514254974 2024-11-25T05:57:42,185 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting f775ce5056b24c0497cf37bbd9ed9951, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732514254974 2024-11-25T05:57:42,186 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting c40046aee2974ff3bd14fee1e595b945, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732514257221 2024-11-25T05:57:42,186 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting b227b28b153841f2b7d8e66511037cb3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732514257221 2024-11-25T05:57:42,186 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting c222fdb5749d47cfa45fb351c787ad9c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732514259390 2024-11-25T05:57:42,186 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting dd0a16b028da43378ef1c338e752124e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732514259390 2024-11-25T05:57:42,190 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:42,191 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125ec8df67e27df4bf997dcd2ac6775ea9b_d756b79bb1dbffd6683a6ced0694b917 store=[table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:42,191 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d756b79bb1dbffd6683a6ced0694b917#B#compaction#470 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:42,192 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/5e96d5fe5ab44b57b227aec97af15802 is 50, key is test_row_0/B:col10/1732514259390/Put/seqid=0 2024-11-25T05:57:42,194 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125ec8df67e27df4bf997dcd2ac6775ea9b_d756b79bb1dbffd6683a6ced0694b917, store=[table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:42,194 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125ec8df67e27df4bf997dcd2ac6775ea9b_d756b79bb1dbffd6683a6ced0694b917 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:42,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742374_1550 (size=12207) 2024-11-25T05:57:42,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742375_1551 (size=4469) 2024-11-25T05:57:42,200 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d756b79bb1dbffd6683a6ced0694b917#A#compaction#471 average throughput is 2.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:42,201 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/97e4b98ea62045d5a0d8e3458996cbc1 is 175, key is test_row_0/A:col10/1732514259390/Put/seqid=0 2024-11-25T05:57:42,202 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/5e96d5fe5ab44b57b227aec97af15802 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/5e96d5fe5ab44b57b227aec97af15802 2024-11-25T05:57:42,208 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d756b79bb1dbffd6683a6ced0694b917/B of d756b79bb1dbffd6683a6ced0694b917 into 5e96d5fe5ab44b57b227aec97af15802(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:42,208 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d756b79bb1dbffd6683a6ced0694b917: 2024-11-25T05:57:42,208 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., storeName=d756b79bb1dbffd6683a6ced0694b917/B, priority=13, startTime=1732514262184; duration=0sec 2024-11-25T05:57:42,208 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:57:42,208 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d756b79bb1dbffd6683a6ced0694b917:B 2024-11-25T05:57:42,208 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:57:42,209 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:57:42,209 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): d756b79bb1dbffd6683a6ced0694b917/C is initiating minor compaction (all files) 2024-11-25T05:57:42,209 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d756b79bb1dbffd6683a6ced0694b917/C in TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:42,209 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/d8a6709cf3e04140b9c9d8be80405073, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/8e9f3bc74bca48709781ea0e4bdb5840, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/ae96ae6839974d42b450eba59ee64697] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp, totalSize=35.3 K 2024-11-25T05:57:42,210 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting d8a6709cf3e04140b9c9d8be80405073, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732514254974 2024-11-25T05:57:42,210 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e9f3bc74bca48709781ea0e4bdb5840, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732514257221 2024-11-25T05:57:42,210 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting ae96ae6839974d42b450eba59ee64697, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732514259390 2024-11-25T05:57:42,219 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d756b79bb1dbffd6683a6ced0694b917#C#compaction#472 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:42,219 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/465eceb14e434c7197f1ee57e8c61ca4 is 50, key is test_row_0/C:col10/1732514259390/Put/seqid=0 2024-11-25T05:57:42,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-25T05:57:42,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742377_1553 (size=12207) 2024-11-25T05:57:42,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742376_1552 (size=31161) 2024-11-25T05:57:42,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-25T05:57:42,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:42,267 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2902): Flushing d756b79bb1dbffd6683a6ced0694b917 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-25T05:57:42,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=A 2024-11-25T05:57:42,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:42,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=B 2024-11-25T05:57:42,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:42,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=C 2024-11-25T05:57:42,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:42,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125872701eecaf34caaa07303800814c76d_d756b79bb1dbffd6683a6ced0694b917 is 50, key is test_row_0/A:col10/1732514260555/Put/seqid=0 2024-11-25T05:57:42,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742378_1554 (size=12154) 2024-11-25T05:57:42,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:57:42,279 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125872701eecaf34caaa07303800814c76d_d756b79bb1dbffd6683a6ced0694b917 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125872701eecaf34caaa07303800814c76d_d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:42,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/761c348de1494430ae65647fe1435eea, store: [table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:42,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/761c348de1494430ae65647fe1435eea is 175, key is test_row_0/A:col10/1732514260555/Put/seqid=0 2024-11-25T05:57:42,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742379_1555 (size=30955) 2024-11-25T05:57:42,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-25T05:57:42,629 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/465eceb14e434c7197f1ee57e8c61ca4 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/465eceb14e434c7197f1ee57e8c61ca4 2024-11-25T05:57:42,632 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d756b79bb1dbffd6683a6ced0694b917/C of d756b79bb1dbffd6683a6ced0694b917 into 465eceb14e434c7197f1ee57e8c61ca4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:42,632 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d756b79bb1dbffd6683a6ced0694b917: 2024-11-25T05:57:42,632 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., storeName=d756b79bb1dbffd6683a6ced0694b917/C, priority=13, startTime=1732514262184; duration=0sec 2024-11-25T05:57:42,632 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:42,632 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d756b79bb1dbffd6683a6ced0694b917:C 2024-11-25T05:57:42,639 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/97e4b98ea62045d5a0d8e3458996cbc1 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/97e4b98ea62045d5a0d8e3458996cbc1 2024-11-25T05:57:42,642 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d756b79bb1dbffd6683a6ced0694b917/A of d756b79bb1dbffd6683a6ced0694b917 into 97e4b98ea62045d5a0d8e3458996cbc1(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:42,642 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d756b79bb1dbffd6683a6ced0694b917: 2024-11-25T05:57:42,642 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., storeName=d756b79bb1dbffd6683a6ced0694b917/A, priority=13, startTime=1732514262184; duration=0sec 2024-11-25T05:57:42,642 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:42,642 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d756b79bb1dbffd6683a6ced0694b917:A 2024-11-25T05:57:42,684 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/761c348de1494430ae65647fe1435eea 2024-11-25T05:57:42,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/f6c3b60d148f4e9a80b834f807c070c2 is 50, key is test_row_0/B:col10/1732514260555/Put/seqid=0 2024-11-25T05:57:42,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742380_1556 (size=12001) 2024-11-25T05:57:42,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. as already flushing 2024-11-25T05:57:42,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:42,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:42,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514322729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:42,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:42,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514322729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:42,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:42,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:42,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:42,732 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:42,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:42,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514322730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:42,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:42,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:42,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:42,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-25T05:57:42,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:42,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514322842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:42,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:42,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:42,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:42,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514322842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:42,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:42,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:42,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514322842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:42,844 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:42,844 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:42,844 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:42,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:42,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:42,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:43,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:43,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514323052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:43,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:43,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:43,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:43,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:43,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514323052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:43,055 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:43,055 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:43,055 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:43,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:43,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514323052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:43,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:43,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:43,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:43,101 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/f6c3b60d148f4e9a80b834f807c070c2 2024-11-25T05:57:43,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/308e1226aee745dca3a80bf8ff21405d is 50, key is test_row_0/C:col10/1732514260555/Put/seqid=0 2024-11-25T05:57:43,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742381_1557 (size=12001) 2024-11-25T05:57:43,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-25T05:57:43,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:43,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:43,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514323362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:43,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514323362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:43,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:43,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514323362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:43,510 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/308e1226aee745dca3a80bf8ff21405d 2024-11-25T05:57:43,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/761c348de1494430ae65647fe1435eea as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/761c348de1494430ae65647fe1435eea 2024-11-25T05:57:43,516 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/761c348de1494430ae65647fe1435eea, entries=150, sequenceid=118, filesize=30.2 K 2024-11-25T05:57:43,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/f6c3b60d148f4e9a80b834f807c070c2 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/f6c3b60d148f4e9a80b834f807c070c2 2024-11-25T05:57:43,518 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/f6c3b60d148f4e9a80b834f807c070c2, entries=150, sequenceid=118, filesize=11.7 K 2024-11-25T05:57:43,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/308e1226aee745dca3a80bf8ff21405d as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/308e1226aee745dca3a80bf8ff21405d 2024-11-25T05:57:43,522 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/308e1226aee745dca3a80bf8ff21405d, entries=150, sequenceid=118, filesize=11.7 K 2024-11-25T05:57:43,523 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for d756b79bb1dbffd6683a6ced0694b917 in 1256ms, sequenceid=118, compaction requested=false 2024-11-25T05:57:43,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2603): Flush status journal for d756b79bb1dbffd6683a6ced0694b917: 2024-11-25T05:57:43,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:43,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=143 2024-11-25T05:57:43,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=143 2024-11-25T05:57:43,525 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-25T05:57:43,525 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4090 sec 2024-11-25T05:57:43,527 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 1.4130 sec 2024-11-25T05:57:43,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:43,875 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d756b79bb1dbffd6683a6ced0694b917 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-25T05:57:43,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=A 2024-11-25T05:57:43,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:43,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=B 2024-11-25T05:57:43,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:43,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=C 2024-11-25T05:57:43,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:43,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125f5115f6cc71844398d60ef8f3d6c4624_d756b79bb1dbffd6683a6ced0694b917 is 50, key is test_row_0/A:col10/1732514263874/Put/seqid=0 2024-11-25T05:57:43,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742382_1558 (size=17284) 2024-11-25T05:57:43,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:43,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514323897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:43,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:43,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:43,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:43,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514323901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:43,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514323902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:43,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:43,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:43,906 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:43,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:43,906 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:43,906 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:44,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:44,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514324011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:44,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:44,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:44,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514324011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:44,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:44,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:44,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:44,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:44,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:44,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:44,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514324012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:44,015 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:44,015 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:44,015 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:44,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:44,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514324222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:44,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:44,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514324222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:44,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:44,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:44,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:44,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:44,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:44,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:44,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:44,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514324223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:44,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:44,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:44,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:44,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-25T05:57:44,262 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:57:44,263 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:57:44,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-25T05:57:44,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-25T05:57:44,264 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:57:44,264 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:57:44,265 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:57:44,285 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:57:44,288 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125f5115f6cc71844398d60ef8f3d6c4624_d756b79bb1dbffd6683a6ced0694b917 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125f5115f6cc71844398d60ef8f3d6c4624_d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:44,288 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/03f9c20376a34388aab80964105bb31f, store: [table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:44,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/03f9c20376a34388aab80964105bb31f is 175, key is test_row_0/A:col10/1732514263874/Put/seqid=0 2024-11-25T05:57:44,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742383_1559 (size=48389) 2024-11-25T05:57:44,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-25T05:57:44,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-25T05:57:44,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:44,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. as already flushing 2024-11-25T05:57:44,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:44,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:44,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:44,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:44,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514324532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:44,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:44,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514324532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:44,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:44,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:44,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:44,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:44,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:44,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:44,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514324532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:44,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:44,536 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:44,536 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:44,536 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:44,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-25T05:57:44,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:44,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. as already flushing 2024-11-25T05:57:44,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:44,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:44,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:44,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:44,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-25T05:57:44,692 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/03f9c20376a34388aab80964105bb31f 2024-11-25T05:57:44,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/799423c5c81649ac8b9edf9a299c6f12 is 50, key is test_row_0/B:col10/1732514263874/Put/seqid=0 2024-11-25T05:57:44,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742384_1560 (size=12151) 2024-11-25T05:57:44,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/799423c5c81649ac8b9edf9a299c6f12 2024-11-25T05:57:44,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/5d1149ce0e944567b8533d9af67ebe6a is 50, key is test_row_0/C:col10/1732514263874/Put/seqid=0 2024-11-25T05:57:44,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-25T05:57:44,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:44,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. as already flushing 2024-11-25T05:57:44,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:44,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:44,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:44,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:44,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742385_1561 (size=12151) 2024-11-25T05:57:44,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-25T05:57:44,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:44,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. as already flushing 2024-11-25T05:57:44,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:44,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:44,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:44,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:44,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-25T05:57:45,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-25T05:57:45,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:45,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. as already flushing 2024-11-25T05:57:45,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:45,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:45,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:45,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:45,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:45,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514325042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:45,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:45,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:45,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514325042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:45,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:45,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:45,044 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:45,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:45,044 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:45,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514325042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:45,044 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:45,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:45,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:45,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:45,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/5d1149ce0e944567b8533d9af67ebe6a 2024-11-25T05:57:45,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/03f9c20376a34388aab80964105bb31f as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/03f9c20376a34388aab80964105bb31f 2024-11-25T05:57:45,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/03f9c20376a34388aab80964105bb31f, entries=250, sequenceid=136, filesize=47.3 K 2024-11-25T05:57:45,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/799423c5c81649ac8b9edf9a299c6f12 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/799423c5c81649ac8b9edf9a299c6f12 2024-11-25T05:57:45,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/799423c5c81649ac8b9edf9a299c6f12, entries=150, sequenceid=136, filesize=11.9 K 2024-11-25T05:57:45,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/5d1149ce0e944567b8533d9af67ebe6a as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/5d1149ce0e944567b8533d9af67ebe6a 2024-11-25T05:57:45,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/5d1149ce0e944567b8533d9af67ebe6a, entries=150, sequenceid=136, filesize=11.9 K 2024-11-25T05:57:45,147 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for d756b79bb1dbffd6683a6ced0694b917 in 1271ms, sequenceid=136, compaction requested=true 2024-11-25T05:57:45,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d756b79bb1dbffd6683a6ced0694b917: 2024-11-25T05:57:45,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d756b79bb1dbffd6683a6ced0694b917:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:57:45,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:45,147 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:57:45,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d756b79bb1dbffd6683a6ced0694b917:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:57:45,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:45,147 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:57:45,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d756b79bb1dbffd6683a6ced0694b917:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:57:45,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:57:45,147 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110505 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:57:45,147 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:57:45,148 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): d756b79bb1dbffd6683a6ced0694b917/A is initiating minor compaction (all files) 2024-11-25T05:57:45,148 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): d756b79bb1dbffd6683a6ced0694b917/B is initiating minor compaction (all files) 2024-11-25T05:57:45,148 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d756b79bb1dbffd6683a6ced0694b917/B in TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:45,148 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d756b79bb1dbffd6683a6ced0694b917/A in TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:45,148 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/97e4b98ea62045d5a0d8e3458996cbc1, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/761c348de1494430ae65647fe1435eea, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/03f9c20376a34388aab80964105bb31f] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp, totalSize=107.9 K 2024-11-25T05:57:45,148 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/5e96d5fe5ab44b57b227aec97af15802, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/f6c3b60d148f4e9a80b834f807c070c2, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/799423c5c81649ac8b9edf9a299c6f12] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp, totalSize=35.5 K 2024-11-25T05:57:45,148 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:45,148 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. files: [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/97e4b98ea62045d5a0d8e3458996cbc1, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/761c348de1494430ae65647fe1435eea, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/03f9c20376a34388aab80964105bb31f] 2024-11-25T05:57:45,148 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e96d5fe5ab44b57b227aec97af15802, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732514259390 2024-11-25T05:57:45,148 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97e4b98ea62045d5a0d8e3458996cbc1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732514259390 2024-11-25T05:57:45,148 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting f6c3b60d148f4e9a80b834f807c070c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732514260555 2024-11-25T05:57:45,148 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 761c348de1494430ae65647fe1435eea, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732514260555 2024-11-25T05:57:45,148 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 799423c5c81649ac8b9edf9a299c6f12, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732514263874 2024-11-25T05:57:45,149 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03f9c20376a34388aab80964105bb31f, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732514262727 2024-11-25T05:57:45,153 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:45,154 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125e6fe1019e5f1484392907d31f6844e74_d756b79bb1dbffd6683a6ced0694b917 store=[table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:45,154 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d756b79bb1dbffd6683a6ced0694b917#B#compaction#479 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:45,155 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/bbb59c03a12042dcb0538acce08ff093 is 50, key is test_row_0/B:col10/1732514263874/Put/seqid=0 2024-11-25T05:57:45,156 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125e6fe1019e5f1484392907d31f6844e74_d756b79bb1dbffd6683a6ced0694b917, store=[table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:45,156 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125e6fe1019e5f1484392907d31f6844e74_d756b79bb1dbffd6683a6ced0694b917 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:45,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742387_1563 (size=4469) 2024-11-25T05:57:45,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742386_1562 (size=12459) 2024-11-25T05:57:45,166 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/bbb59c03a12042dcb0538acce08ff093 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/bbb59c03a12042dcb0538acce08ff093 2024-11-25T05:57:45,170 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d756b79bb1dbffd6683a6ced0694b917/B of d756b79bb1dbffd6683a6ced0694b917 into bbb59c03a12042dcb0538acce08ff093(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:45,170 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d756b79bb1dbffd6683a6ced0694b917: 2024-11-25T05:57:45,170 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., storeName=d756b79bb1dbffd6683a6ced0694b917/B, priority=13, startTime=1732514265147; duration=0sec 2024-11-25T05:57:45,170 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:57:45,170 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d756b79bb1dbffd6683a6ced0694b917:B 2024-11-25T05:57:45,170 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:57:45,170 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:57:45,170 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): d756b79bb1dbffd6683a6ced0694b917/C is initiating minor compaction (all files) 2024-11-25T05:57:45,170 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d756b79bb1dbffd6683a6ced0694b917/C in TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:45,171 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/465eceb14e434c7197f1ee57e8c61ca4, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/308e1226aee745dca3a80bf8ff21405d, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/5d1149ce0e944567b8533d9af67ebe6a] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp, totalSize=35.5 K 2024-11-25T05:57:45,171 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 465eceb14e434c7197f1ee57e8c61ca4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732514259390 2024-11-25T05:57:45,171 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 308e1226aee745dca3a80bf8ff21405d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732514260555 2024-11-25T05:57:45,172 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d1149ce0e944567b8533d9af67ebe6a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732514263874 2024-11-25T05:57:45,176 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d756b79bb1dbffd6683a6ced0694b917#C#compaction#481 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:45,176 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/67e0d12f91e8447c97888974adf5e696 is 50, key is test_row_0/C:col10/1732514263874/Put/seqid=0 2024-11-25T05:57:45,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-25T05:57:45,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:45,179 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing d756b79bb1dbffd6683a6ced0694b917 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-25T05:57:45,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742388_1564 (size=12459) 2024-11-25T05:57:45,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=A 2024-11-25T05:57:45,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:45,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=B 2024-11-25T05:57:45,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:45,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=C 2024-11-25T05:57:45,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:45,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125bb206d45b1644e89baa88500a697a7c4_d756b79bb1dbffd6683a6ced0694b917 is 50, key is test_row_0/A:col10/1732514263895/Put/seqid=0 2024-11-25T05:57:45,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742389_1565 (size=12304) 2024-11-25T05:57:45,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-25T05:57:45,563 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d756b79bb1dbffd6683a6ced0694b917#A#compaction#480 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:45,564 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/88f4507e1d804a0caff3fae07a86de77 is 175, key is test_row_0/A:col10/1732514263874/Put/seqid=0 2024-11-25T05:57:45,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742390_1566 (size=31413) 2024-11-25T05:57:45,583 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/67e0d12f91e8447c97888974adf5e696 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/67e0d12f91e8447c97888974adf5e696 2024-11-25T05:57:45,587 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d756b79bb1dbffd6683a6ced0694b917/C of d756b79bb1dbffd6683a6ced0694b917 into 67e0d12f91e8447c97888974adf5e696(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:45,587 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d756b79bb1dbffd6683a6ced0694b917: 2024-11-25T05:57:45,587 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., storeName=d756b79bb1dbffd6683a6ced0694b917/C, priority=13, startTime=1732514265147; duration=0sec 2024-11-25T05:57:45,587 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:45,587 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d756b79bb1dbffd6683a6ced0694b917:C 2024-11-25T05:57:45,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:57:45,594 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125bb206d45b1644e89baa88500a697a7c4_d756b79bb1dbffd6683a6ced0694b917 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125bb206d45b1644e89baa88500a697a7c4_d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:45,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/5da9fb984da44bb99cfe501a62c74e1a, store: [table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:45,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/5da9fb984da44bb99cfe501a62c74e1a is 175, key is test_row_0/A:col10/1732514263895/Put/seqid=0 2024-11-25T05:57:45,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742391_1567 (size=31105) 2024-11-25T05:57:45,970 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/88f4507e1d804a0caff3fae07a86de77 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/88f4507e1d804a0caff3fae07a86de77 2024-11-25T05:57:45,973 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d756b79bb1dbffd6683a6ced0694b917/A of d756b79bb1dbffd6683a6ced0694b917 into 88f4507e1d804a0caff3fae07a86de77(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:45,973 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d756b79bb1dbffd6683a6ced0694b917: 2024-11-25T05:57:45,973 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., storeName=d756b79bb1dbffd6683a6ced0694b917/A, priority=13, startTime=1732514265147; duration=0sec 2024-11-25T05:57:45,974 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:45,974 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d756b79bb1dbffd6683a6ced0694b917:A 2024-11-25T05:57:45,999 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=154, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/5da9fb984da44bb99cfe501a62c74e1a 2024-11-25T05:57:46,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/f9e66aacdd7d4b9f9a175a4c98173fb9 is 50, key is test_row_0/B:col10/1732514263895/Put/seqid=0 2024-11-25T05:57:46,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742392_1568 (size=12151) 2024-11-25T05:57:46,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. as already flushing 2024-11-25T05:57:46,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:46,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:46,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514326069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:46,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:46,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:46,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:46,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:46,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514326074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:46,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:46,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514326074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:46,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:46,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514326182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:46,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:46,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:46,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514326182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:46,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:46,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:46,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:46,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:46,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:46,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:46,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514326182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:46,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:46,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:46,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:46,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:46,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514326392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:46,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:46,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514326392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:46,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:46,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:46,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514326392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:46,395 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:46,395 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:46,395 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:46,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:46,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:46,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:46,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-25T05:57:46,419 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/f9e66aacdd7d4b9f9a175a4c98173fb9 2024-11-25T05:57:46,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/51645033018f427eb8e60ded5443229e is 50, key is test_row_0/C:col10/1732514263895/Put/seqid=0 2024-11-25T05:57:46,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742393_1569 (size=12151) 2024-11-25T05:57:46,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:46,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514326702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:46,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:46,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:46,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:46,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:46,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514326702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:46,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:46,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:46,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:46,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:46,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514326702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:46,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:46,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:46,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:46,846 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/51645033018f427eb8e60ded5443229e 2024-11-25T05:57:46,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/5da9fb984da44bb99cfe501a62c74e1a as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/5da9fb984da44bb99cfe501a62c74e1a 2024-11-25T05:57:46,853 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/5da9fb984da44bb99cfe501a62c74e1a, entries=150, sequenceid=154, filesize=30.4 K 2024-11-25T05:57:46,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/f9e66aacdd7d4b9f9a175a4c98173fb9 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/f9e66aacdd7d4b9f9a175a4c98173fb9 2024-11-25T05:57:46,856 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/f9e66aacdd7d4b9f9a175a4c98173fb9, entries=150, sequenceid=154, filesize=11.9 K 2024-11-25T05:57:46,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/51645033018f427eb8e60ded5443229e as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/51645033018f427eb8e60ded5443229e 2024-11-25T05:57:46,859 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/51645033018f427eb8e60ded5443229e, entries=150, sequenceid=154, filesize=11.9 K 2024-11-25T05:57:46,860 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for d756b79bb1dbffd6683a6ced0694b917 in 1681ms, sequenceid=154, compaction requested=false 2024-11-25T05:57:46,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for d756b79bb1dbffd6683a6ced0694b917: 2024-11-25T05:57:46,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:46,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-25T05:57:46,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-25T05:57:46,863 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-25T05:57:46,863 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5970 sec 2024-11-25T05:57:46,864 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 2.6000 sec 2024-11-25T05:57:47,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:47,213 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d756b79bb1dbffd6683a6ced0694b917 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-25T05:57:47,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=A 2024-11-25T05:57:47,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:47,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=B 2024-11-25T05:57:47,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:47,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=C 2024-11-25T05:57:47,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:47,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125d22e7108cfcc4acbbb7499974d7bf9f7_d756b79bb1dbffd6683a6ced0694b917 is 50, key is test_row_0/A:col10/1732514266065/Put/seqid=0 2024-11-25T05:57:47,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742394_1570 (size=14794) 2024-11-25T05:57:47,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:47,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514327231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:47,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:47,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:47,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:47,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:47,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514327237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:47,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:47,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:47,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:47,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:47,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514327237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:47,243 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:47,243 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:47,243 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:47,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:47,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514327342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:47,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:47,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:47,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:47,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:47,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514327352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:47,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:47,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:47,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514327352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:47,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:47,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:47,355 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:47,355 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:47,355 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:47,546 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T05:57:47,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514327552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:47,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:47,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:47,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:47,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514327562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:47,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:47,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:47,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:47,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514327562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:47,563 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:47,622 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:57:47,624 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125d22e7108cfcc4acbbb7499974d7bf9f7_d756b79bb1dbffd6683a6ced0694b917 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d22e7108cfcc4acbbb7499974d7bf9f7_d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:47,625 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/672c8f3b2d1a4ab5ac0b7631c0bf67c7, store: [table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:47,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/672c8f3b2d1a4ab5ac0b7631c0bf67c7 is 175, key is test_row_0/A:col10/1732514266065/Put/seqid=0 2024-11-25T05:57:47,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742395_1571 (size=39749) 2024-11-25T05:57:47,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:47,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514327862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:47,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:47,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:47,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:47,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:47,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:47,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514327872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:47,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514327872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:47,876 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:47,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:47,876 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:47,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:47,876 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:47,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:48,028 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/672c8f3b2d1a4ab5ac0b7631c0bf67c7 2024-11-25T05:57:48,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/3f76e5b216b041fab7883897e14f7a24 is 50, key is test_row_0/B:col10/1732514266065/Put/seqid=0 2024-11-25T05:57:48,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742396_1572 (size=12151) 2024-11-25T05:57:48,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:48,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514328372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:48,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:48,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:48,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:48,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:48,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514328382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:48,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:48,385 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:48,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514328382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:48,385 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:48,385 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:48,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:48,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:48,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:48,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-25T05:57:48,432 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:57:48,432 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:57:48,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-25T05:57:48,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-25T05:57:48,434 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:57:48,434 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:57:48,434 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:57:48,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/3f76e5b216b041fab7883897e14f7a24 2024-11-25T05:57:48,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/e1600ebc2a7748f8990657888c0d8060 is 50, key is test_row_0/C:col10/1732514266065/Put/seqid=0 2024-11-25T05:57:48,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742397_1573 (size=12151) 2024-11-25T05:57:48,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-25T05:57:48,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-25T05:57:48,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:48,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. as already flushing 2024-11-25T05:57:48,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:48,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:48,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:48,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:48,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-25T05:57:48,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:48,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. as already flushing 2024-11-25T05:57:48,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:48,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:48,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:48,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:48,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-25T05:57:48,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/e1600ebc2a7748f8990657888c0d8060 2024-11-25T05:57:48,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/672c8f3b2d1a4ab5ac0b7631c0bf67c7 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/672c8f3b2d1a4ab5ac0b7631c0bf67c7 2024-11-25T05:57:48,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/672c8f3b2d1a4ab5ac0b7631c0bf67c7, entries=200, sequenceid=172, filesize=38.8 K 2024-11-25T05:57:48,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/3f76e5b216b041fab7883897e14f7a24 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/3f76e5b216b041fab7883897e14f7a24 2024-11-25T05:57:48,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/3f76e5b216b041fab7883897e14f7a24, entries=150, sequenceid=172, filesize=11.9 K 2024-11-25T05:57:48,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/e1600ebc2a7748f8990657888c0d8060 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/e1600ebc2a7748f8990657888c0d8060 2024-11-25T05:57:48,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/e1600ebc2a7748f8990657888c0d8060, entries=150, sequenceid=172, filesize=11.9 K 2024-11-25T05:57:48,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for d756b79bb1dbffd6683a6ced0694b917 in 1645ms, sequenceid=172, compaction requested=true 2024-11-25T05:57:48,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d756b79bb1dbffd6683a6ced0694b917: 2024-11-25T05:57:48,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d756b79bb1dbffd6683a6ced0694b917:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:57:48,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:48,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d756b79bb1dbffd6683a6ced0694b917:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:57:48,859 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:57:48,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:48,859 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:57:48,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d756b79bb1dbffd6683a6ced0694b917:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:57:48,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:57:48,859 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:57:48,859 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:57:48,859 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): d756b79bb1dbffd6683a6ced0694b917/A is initiating minor compaction (all files) 2024-11-25T05:57:48,859 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): d756b79bb1dbffd6683a6ced0694b917/B is initiating minor compaction (all files) 2024-11-25T05:57:48,859 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d756b79bb1dbffd6683a6ced0694b917/B in TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:48,859 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d756b79bb1dbffd6683a6ced0694b917/A in TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:48,860 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/88f4507e1d804a0caff3fae07a86de77, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/5da9fb984da44bb99cfe501a62c74e1a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/672c8f3b2d1a4ab5ac0b7631c0bf67c7] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp, totalSize=99.9 K 2024-11-25T05:57:48,860 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/bbb59c03a12042dcb0538acce08ff093, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/f9e66aacdd7d4b9f9a175a4c98173fb9, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/3f76e5b216b041fab7883897e14f7a24] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp, totalSize=35.9 K 2024-11-25T05:57:48,860 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:48,860 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. files: [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/88f4507e1d804a0caff3fae07a86de77, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/5da9fb984da44bb99cfe501a62c74e1a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/672c8f3b2d1a4ab5ac0b7631c0bf67c7] 2024-11-25T05:57:48,860 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting bbb59c03a12042dcb0538acce08ff093, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732514263874 2024-11-25T05:57:48,860 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88f4507e1d804a0caff3fae07a86de77, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732514263874 2024-11-25T05:57:48,860 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting f9e66aacdd7d4b9f9a175a4c98173fb9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732514263895 2024-11-25T05:57:48,860 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5da9fb984da44bb99cfe501a62c74e1a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732514263895 2024-11-25T05:57:48,860 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f76e5b216b041fab7883897e14f7a24, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732514266065 2024-11-25T05:57:48,860 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 672c8f3b2d1a4ab5ac0b7631c0bf67c7, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732514266065 2024-11-25T05:57:48,864 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:48,866 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d756b79bb1dbffd6683a6ced0694b917#B#compaction#488 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:48,866 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/9ed00792d9d04a5e8aa63d593321debe is 50, key is test_row_0/B:col10/1732514266065/Put/seqid=0 2024-11-25T05:57:48,867 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125a3268208319441e1888c1c322a58f34e_d756b79bb1dbffd6683a6ced0694b917 store=[table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:48,869 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125a3268208319441e1888c1c322a58f34e_d756b79bb1dbffd6683a6ced0694b917, store=[table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:48,870 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125a3268208319441e1888c1c322a58f34e_d756b79bb1dbffd6683a6ced0694b917 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:48,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742398_1574 (size=12561) 2024-11-25T05:57:48,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742399_1575 (size=4469) 2024-11-25T05:57:48,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-25T05:57:48,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:48,892 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing d756b79bb1dbffd6683a6ced0694b917 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-25T05:57:48,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=A 2024-11-25T05:57:48,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:48,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=B 2024-11-25T05:57:48,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:48,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=C 2024-11-25T05:57:48,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:48,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411256cf2ff83287146b886d0ea16014dfd24_d756b79bb1dbffd6683a6ced0694b917 is 50, key is test_row_0/A:col10/1732514267236/Put/seqid=0 2024-11-25T05:57:48,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742400_1576 (size=12304) 2024-11-25T05:57:49,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-25T05:57:49,275 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/9ed00792d9d04a5e8aa63d593321debe as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/9ed00792d9d04a5e8aa63d593321debe 2024-11-25T05:57:49,275 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d756b79bb1dbffd6683a6ced0694b917#A#compaction#489 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:49,276 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/e7176885638445808b2f06cd3c6c52b4 is 175, key is test_row_0/A:col10/1732514266065/Put/seqid=0 2024-11-25T05:57:49,278 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d756b79bb1dbffd6683a6ced0694b917/B of d756b79bb1dbffd6683a6ced0694b917 into 9ed00792d9d04a5e8aa63d593321debe(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:49,279 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d756b79bb1dbffd6683a6ced0694b917: 2024-11-25T05:57:49,279 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., storeName=d756b79bb1dbffd6683a6ced0694b917/B, priority=13, startTime=1732514268859; duration=0sec 2024-11-25T05:57:49,279 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:57:49,279 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d756b79bb1dbffd6683a6ced0694b917:B 2024-11-25T05:57:49,279 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:57:49,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742401_1577 (size=31515) 2024-11-25T05:57:49,280 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:57:49,280 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): d756b79bb1dbffd6683a6ced0694b917/C is initiating minor compaction (all files) 2024-11-25T05:57:49,280 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d756b79bb1dbffd6683a6ced0694b917/C in TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:49,280 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/67e0d12f91e8447c97888974adf5e696, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/51645033018f427eb8e60ded5443229e, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/e1600ebc2a7748f8990657888c0d8060] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp, totalSize=35.9 K 2024-11-25T05:57:49,280 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 67e0d12f91e8447c97888974adf5e696, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732514263874 2024-11-25T05:57:49,281 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 51645033018f427eb8e60ded5443229e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732514263895 2024-11-25T05:57:49,281 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting e1600ebc2a7748f8990657888c0d8060, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732514266065 2024-11-25T05:57:49,286 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d756b79bb1dbffd6683a6ced0694b917#C#compaction#491 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:49,286 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/c7e0580c9d344c00aa826f0c1bf8b622 is 50, key is test_row_0/C:col10/1732514266065/Put/seqid=0 2024-11-25T05:57:49,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742402_1578 (size=12561) 2024-11-25T05:57:49,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:57:49,317 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411256cf2ff83287146b886d0ea16014dfd24_d756b79bb1dbffd6683a6ced0694b917 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256cf2ff83287146b886d0ea16014dfd24_d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:49,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/31781db0e2574399832e47fbfc830976, store: [table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:49,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/31781db0e2574399832e47fbfc830976 is 175, key is test_row_0/A:col10/1732514267236/Put/seqid=0 2024-11-25T05:57:49,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742403_1579 (size=31105) 2024-11-25T05:57:49,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:49,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. as already flushing 2024-11-25T05:57:49,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:49,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514329409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:49,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:49,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:49,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:49,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:49,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514329410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:49,415 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:49,415 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:49,415 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:49,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:49,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514329411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:49,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:49,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:49,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:49,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:49,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514329522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:49,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:49,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:49,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:49,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:49,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514329522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:49,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:49,526 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:49,526 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:49,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514329522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:49,526 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:49,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:49,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:49,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:49,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-25T05:57:49,683 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/e7176885638445808b2f06cd3c6c52b4 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/e7176885638445808b2f06cd3c6c52b4 2024-11-25T05:57:49,686 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d756b79bb1dbffd6683a6ced0694b917/A of d756b79bb1dbffd6683a6ced0694b917 into e7176885638445808b2f06cd3c6c52b4(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:49,686 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d756b79bb1dbffd6683a6ced0694b917: 2024-11-25T05:57:49,686 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., storeName=d756b79bb1dbffd6683a6ced0694b917/A, priority=13, startTime=1732514268859; duration=0sec 2024-11-25T05:57:49,687 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:49,687 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d756b79bb1dbffd6683a6ced0694b917:A 2024-11-25T05:57:49,697 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/c7e0580c9d344c00aa826f0c1bf8b622 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/c7e0580c9d344c00aa826f0c1bf8b622 2024-11-25T05:57:49,701 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d756b79bb1dbffd6683a6ced0694b917/C of d756b79bb1dbffd6683a6ced0694b917 into c7e0580c9d344c00aa826f0c1bf8b622(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:49,701 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d756b79bb1dbffd6683a6ced0694b917: 2024-11-25T05:57:49,701 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., storeName=d756b79bb1dbffd6683a6ced0694b917/C, priority=13, startTime=1732514268859; duration=0sec 2024-11-25T05:57:49,701 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:49,701 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d756b79bb1dbffd6683a6ced0694b917:C 2024-11-25T05:57:49,721 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=189, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/31781db0e2574399832e47fbfc830976 2024-11-25T05:57:49,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/332f982340f94a1cba2a5be0d75a1181 is 50, key is test_row_0/B:col10/1732514267236/Put/seqid=0 2024-11-25T05:57:49,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742404_1580 (size=12151) 2024-11-25T05:57:49,730 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/332f982340f94a1cba2a5be0d75a1181 2024-11-25T05:57:49,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/bcdb4e89e7604e1896e22a49bb56f3a5 is 50, key is test_row_0/C:col10/1732514267236/Put/seqid=0 2024-11-25T05:57:49,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:49,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514329732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:49,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:49,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:49,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514329732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:49,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:49,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:49,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:49,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514329732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:49,736 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:49,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:49,736 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:49,736 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:49,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:49,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:49,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742405_1581 (size=12151) 2024-11-25T05:57:50,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:50,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514330042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:50,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:50,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:50,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514330042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:50,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:50,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:50,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:50,044 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:50,044 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:50,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514330042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:50,044 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:50,138 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/bcdb4e89e7604e1896e22a49bb56f3a5 2024-11-25T05:57:50,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/31781db0e2574399832e47fbfc830976 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/31781db0e2574399832e47fbfc830976 2024-11-25T05:57:50,143 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/31781db0e2574399832e47fbfc830976, entries=150, sequenceid=189, filesize=30.4 K 2024-11-25T05:57:50,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/332f982340f94a1cba2a5be0d75a1181 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/332f982340f94a1cba2a5be0d75a1181 2024-11-25T05:57:50,146 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/332f982340f94a1cba2a5be0d75a1181, entries=150, sequenceid=189, filesize=11.9 K 2024-11-25T05:57:50,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/bcdb4e89e7604e1896e22a49bb56f3a5 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/bcdb4e89e7604e1896e22a49bb56f3a5 2024-11-25T05:57:50,149 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/bcdb4e89e7604e1896e22a49bb56f3a5, entries=150, sequenceid=189, filesize=11.9 K 2024-11-25T05:57:50,150 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for d756b79bb1dbffd6683a6ced0694b917 in 1258ms, sequenceid=189, compaction requested=false 2024-11-25T05:57:50,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for d756b79bb1dbffd6683a6ced0694b917: 2024-11-25T05:57:50,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:50,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-25T05:57:50,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-25T05:57:50,152 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-25T05:57:50,152 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7170 sec 2024-11-25T05:57:50,154 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 1.7210 sec 2024-11-25T05:57:50,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:50,554 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d756b79bb1dbffd6683a6ced0694b917 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-25T05:57:50,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=A 2024-11-25T05:57:50,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:50,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=B 2024-11-25T05:57:50,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:50,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=C 2024-11-25T05:57:50,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:50,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411257ac16e1802f1416881b5a1a001ee687e_d756b79bb1dbffd6683a6ced0694b917 is 50, key is test_row_0/A:col10/1732514269409/Put/seqid=0 2024-11-25T05:57:50,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742406_1582 (size=14794) 2024-11-25T05:57:50,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:50,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514330573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:50,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:50,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:50,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:50,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:50,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514330576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:50,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:50,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514330576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:50,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:50,580 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:50,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:50,580 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:50,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:50,580 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:50,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-25T05:57:50,582 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:57:50,583 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-25T05:57:50,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-25T05:57:50,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T05:57:50,584 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T05:57:50,585 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T05:57:50,585 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T05:57:50,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:50,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514330682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:50,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:50,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514330682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:50,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:50,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:50,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:50,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:50,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:50,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:50,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514330682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:50,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:50,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:50,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:50,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:50,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T05:57:50,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T05:57:50,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:50,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. as already flushing 2024-11-25T05:57:50,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:50,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:50,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:50,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T05:57:50,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:50,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. as already flushing 2024-11-25T05:57:50,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:50,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:50,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:50,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:50,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:50,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514330892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:50,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:50,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514330892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:50,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:50,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:50,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:50,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:50,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:50,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514330892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:50,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:50,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:50,894 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:50,894 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:50,894 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:50,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T05:57:50,963 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:57:50,965 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411257ac16e1802f1416881b5a1a001ee687e_d756b79bb1dbffd6683a6ced0694b917 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411257ac16e1802f1416881b5a1a001ee687e_d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:50,966 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/1f53283c5a564f5db7f0889ebf60f8bd, store: [table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:50,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/1f53283c5a564f5db7f0889ebf60f8bd is 175, key is test_row_0/A:col10/1732514269409/Put/seqid=0 2024-11-25T05:57:50,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742407_1583 (size=39749) 2024-11-25T05:57:51,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T05:57:51,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:51,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. as already flushing 2024-11-25T05:57:51,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:51,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:51,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:51,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:51,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T05:57:51,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:51,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. as already flushing 2024-11-25T05:57:51,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:51,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:51,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:51,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:51,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:51,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514331202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:51,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:51,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:51,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514331202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:51,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:51,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:51,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:51,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:51,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514331202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:51,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:51,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:51,205 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:51,205 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:51,205 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T05:57:51,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T05:57:51,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:51,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. as already flushing 2024-11-25T05:57:51,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:51,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:51,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:51,370 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/1f53283c5a564f5db7f0889ebf60f8bd 2024-11-25T05:57:51,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/8e2df617a9084035af3502c7f2e85e94 is 50, key is test_row_0/B:col10/1732514269409/Put/seqid=0 2024-11-25T05:57:51,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742408_1584 (size=12151) 2024-11-25T05:57:51,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T05:57:51,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:51,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. as already flushing 2024-11-25T05:57:51,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:51,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:51,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:51,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T05:57:51,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:51,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. as already flushing 2024-11-25T05:57:51,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:51,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:51,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:51,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514331712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:51,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:51,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:51,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514331712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:51,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:51,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:51,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:51,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514331712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:51,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:51,715 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:51,715 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:51,715 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T05:57:51,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/8e2df617a9084035af3502c7f2e85e94 2024-11-25T05:57:51,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/748d67be63e54be589bffd2b00df8a3a is 50, key is test_row_0/C:col10/1732514269409/Put/seqid=0 2024-11-25T05:57:51,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742409_1585 (size=12151) 2024-11-25T05:57:51,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T05:57:51,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:51,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. as already flushing 2024-11-25T05:57:51,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:51,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:51,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:51,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:51,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T05:57:51,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:51,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. as already flushing 2024-11-25T05:57:51,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:51,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:51,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:51,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:52,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T05:57:52,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:52,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. as already flushing 2024-11-25T05:57:52,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:52,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:52,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:52,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T05:57:52,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:52,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57984 deadline: 1732514332132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:52,133 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_2' in d756b79bb1dbffd6683a6ced0694b917 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18296 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:57:52,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:52,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:52,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:52,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:52,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58000 deadline: 1732514332132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:52,135 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 8ef925b832e3,35599,1732514089005 for 'test_row_2' in d756b79bb1dbffd6683a6ced0694b917 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18298 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T05:57:52,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:52,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:52,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:52,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/748d67be63e54be589bffd2b00df8a3a 2024-11-25T05:57:52,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/1f53283c5a564f5db7f0889ebf60f8bd as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/1f53283c5a564f5db7f0889ebf60f8bd 2024-11-25T05:57:52,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/1f53283c5a564f5db7f0889ebf60f8bd, entries=200, sequenceid=208, filesize=38.8 K 2024-11-25T05:57:52,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/8e2df617a9084035af3502c7f2e85e94 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/8e2df617a9084035af3502c7f2e85e94 2024-11-25T05:57:52,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/8e2df617a9084035af3502c7f2e85e94, entries=150, sequenceid=208, filesize=11.9 K 2024-11-25T05:57:52,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/748d67be63e54be589bffd2b00df8a3a as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/748d67be63e54be589bffd2b00df8a3a 2024-11-25T05:57:52,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/748d67be63e54be589bffd2b00df8a3a, entries=150, sequenceid=208, filesize=11.9 K 2024-11-25T05:57:52,201 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for d756b79bb1dbffd6683a6ced0694b917 in 1647ms, sequenceid=208, compaction requested=true 2024-11-25T05:57:52,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d756b79bb1dbffd6683a6ced0694b917: 2024-11-25T05:57:52,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d756b79bb1dbffd6683a6ced0694b917:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:57:52,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:52,201 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:57:52,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d756b79bb1dbffd6683a6ced0694b917:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:57:52,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:52,201 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:57:52,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d756b79bb1dbffd6683a6ced0694b917:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:57:52,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:57:52,202 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:57:52,202 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:57:52,202 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): d756b79bb1dbffd6683a6ced0694b917/B is initiating minor compaction (all files) 2024-11-25T05:57:52,202 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): d756b79bb1dbffd6683a6ced0694b917/A is initiating minor compaction (all files) 2024-11-25T05:57:52,202 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d756b79bb1dbffd6683a6ced0694b917/B in TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:52,202 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d756b79bb1dbffd6683a6ced0694b917/A in TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:52,202 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/9ed00792d9d04a5e8aa63d593321debe, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/332f982340f94a1cba2a5be0d75a1181, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/8e2df617a9084035af3502c7f2e85e94] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp, totalSize=36.0 K 2024-11-25T05:57:52,202 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/e7176885638445808b2f06cd3c6c52b4, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/31781db0e2574399832e47fbfc830976, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/1f53283c5a564f5db7f0889ebf60f8bd] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp, totalSize=100.0 K 2024-11-25T05:57:52,202 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:52,202 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. files: [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/e7176885638445808b2f06cd3c6c52b4, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/31781db0e2574399832e47fbfc830976, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/1f53283c5a564f5db7f0889ebf60f8bd] 2024-11-25T05:57:52,202 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ed00792d9d04a5e8aa63d593321debe, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732514266065 2024-11-25T05:57:52,203 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7176885638445808b2f06cd3c6c52b4, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732514266065 2024-11-25T05:57:52,203 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 332f982340f94a1cba2a5be0d75a1181, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732514267222 2024-11-25T05:57:52,203 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31781db0e2574399832e47fbfc830976, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732514267222 2024-11-25T05:57:52,203 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e2df617a9084035af3502c7f2e85e94, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732514269408 2024-11-25T05:57:52,203 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f53283c5a564f5db7f0889ebf60f8bd, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732514269408 2024-11-25T05:57:52,209 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:52,210 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d756b79bb1dbffd6683a6ced0694b917#B#compaction#497 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:52,210 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/402f6017e2804d74b7f57fb36f6db873 is 50, key is test_row_0/B:col10/1732514269409/Put/seqid=0 2024-11-25T05:57:52,218 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125b8eec3a7f88c48a591a3d4fbc464c1e1_d756b79bb1dbffd6683a6ced0694b917 store=[table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:52,220 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125b8eec3a7f88c48a591a3d4fbc464c1e1_d756b79bb1dbffd6683a6ced0694b917, store=[table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:52,220 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125b8eec3a7f88c48a591a3d4fbc464c1e1_d756b79bb1dbffd6683a6ced0694b917 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:52,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742410_1586 (size=12663) 2024-11-25T05:57:52,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742411_1587 (size=4469) 2024-11-25T05:57:52,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35599 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T05:57:52,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:52,262 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing d756b79bb1dbffd6683a6ced0694b917 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-25T05:57:52,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=A 2024-11-25T05:57:52,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:52,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=B 2024-11-25T05:57:52,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:52,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=C 2024-11-25T05:57:52,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:52,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411250ee2d1c3b7c04351b87317513f12396f_d756b79bb1dbffd6683a6ced0694b917 is 50, key is test_row_0/A:col10/1732514270575/Put/seqid=0 2024-11-25T05:57:52,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742412_1588 (size=12304) 2024-11-25T05:57:52,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:57:52,279 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411250ee2d1c3b7c04351b87317513f12396f_d756b79bb1dbffd6683a6ced0694b917 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250ee2d1c3b7c04351b87317513f12396f_d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:52,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/bc7de207d0a74b228e8c3d38ec6562b1, store: [table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:52,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/bc7de207d0a74b228e8c3d38ec6562b1 is 175, key is test_row_0/A:col10/1732514270575/Put/seqid=0 2024-11-25T05:57:52,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742413_1589 (size=31105) 2024-11-25T05:57:52,630 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/402f6017e2804d74b7f57fb36f6db873 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/402f6017e2804d74b7f57fb36f6db873 2024-11-25T05:57:52,633 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d756b79bb1dbffd6683a6ced0694b917/B of d756b79bb1dbffd6683a6ced0694b917 into 402f6017e2804d74b7f57fb36f6db873(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:52,634 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d756b79bb1dbffd6683a6ced0694b917: 2024-11-25T05:57:52,634 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., storeName=d756b79bb1dbffd6683a6ced0694b917/B, priority=13, startTime=1732514272201; duration=0sec 2024-11-25T05:57:52,634 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:57:52,634 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d756b79bb1dbffd6683a6ced0694b917:B 2024-11-25T05:57:52,634 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:57:52,635 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:57:52,635 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d756b79bb1dbffd6683a6ced0694b917#A#compaction#498 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:52,635 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): d756b79bb1dbffd6683a6ced0694b917/C is initiating minor compaction (all files) 2024-11-25T05:57:52,635 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d756b79bb1dbffd6683a6ced0694b917/C in TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:52,635 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/c7e0580c9d344c00aa826f0c1bf8b622, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/bcdb4e89e7604e1896e22a49bb56f3a5, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/748d67be63e54be589bffd2b00df8a3a] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp, totalSize=36.0 K 2024-11-25T05:57:52,635 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/c970d85ef3f14aa4b0db34a404697685 is 175, key is test_row_0/A:col10/1732514269409/Put/seqid=0 2024-11-25T05:57:52,635 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting c7e0580c9d344c00aa826f0c1bf8b622, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732514266065 2024-11-25T05:57:52,636 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting bcdb4e89e7604e1896e22a49bb56f3a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732514267222 2024-11-25T05:57:52,636 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 748d67be63e54be589bffd2b00df8a3a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732514269408 2024-11-25T05:57:52,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742414_1590 (size=31617) 2024-11-25T05:57:52,642 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d756b79bb1dbffd6683a6ced0694b917#C#compaction#500 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:52,643 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/6371173559d342ac845efcaa5d001b55 is 50, key is test_row_0/C:col10/1732514269409/Put/seqid=0 2024-11-25T05:57:52,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742415_1591 (size=12663) 2024-11-25T05:57:52,684 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=225, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/bc7de207d0a74b228e8c3d38ec6562b1 2024-11-25T05:57:52,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/81ccbeb749354e5b99a817ec5e6375e9 is 50, key is test_row_0/B:col10/1732514270575/Put/seqid=0 2024-11-25T05:57:52,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742416_1592 (size=12151) 2024-11-25T05:57:52,695 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/81ccbeb749354e5b99a817ec5e6375e9 2024-11-25T05:57:52,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/a4bd4cb3470f40319d8353509cc5c9c0 is 50, key is test_row_0/C:col10/1732514270575/Put/seqid=0 2024-11-25T05:57:52,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742417_1593 (size=12151) 2024-11-25T05:57:52,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. as already flushing 2024-11-25T05:57:52,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:52,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T05:57:52,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514332746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:52,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514332746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:52,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:52,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514332747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:52,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:52,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:52,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:52,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:52,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514332862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:52,865 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:52,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:52,865 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:52,865 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:52,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514332862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:52,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:52,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:52,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:52,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:52,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514332862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:52,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:52,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:52,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:53,044 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/c970d85ef3f14aa4b0db34a404697685 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/c970d85ef3f14aa4b0db34a404697685 2024-11-25T05:57:53,048 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d756b79bb1dbffd6683a6ced0694b917/A of d756b79bb1dbffd6683a6ced0694b917 into c970d85ef3f14aa4b0db34a404697685(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:53,048 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d756b79bb1dbffd6683a6ced0694b917: 2024-11-25T05:57:53,048 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., storeName=d756b79bb1dbffd6683a6ced0694b917/A, priority=13, startTime=1732514272201; duration=0sec 2024-11-25T05:57:53,048 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:53,048 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d756b79bb1dbffd6683a6ced0694b917:A 2024-11-25T05:57:53,049 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/6371173559d342ac845efcaa5d001b55 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/6371173559d342ac845efcaa5d001b55 2024-11-25T05:57:53,054 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d756b79bb1dbffd6683a6ced0694b917/C of d756b79bb1dbffd6683a6ced0694b917 into 6371173559d342ac845efcaa5d001b55(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:53,054 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d756b79bb1dbffd6683a6ced0694b917: 2024-11-25T05:57:53,054 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., storeName=d756b79bb1dbffd6683a6ced0694b917/C, priority=13, startTime=1732514272201; duration=0sec 2024-11-25T05:57:53,054 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:53,054 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d756b79bb1dbffd6683a6ced0694b917:C 2024-11-25T05:57:53,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:53,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514333072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:53,076 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:53,076 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:53,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:53,076 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:53,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514333072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:53,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:53,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:53,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:53,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:53,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514333073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:53,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:53,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:53,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:53,107 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/a4bd4cb3470f40319d8353509cc5c9c0 2024-11-25T05:57:53,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/bc7de207d0a74b228e8c3d38ec6562b1 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/bc7de207d0a74b228e8c3d38ec6562b1 2024-11-25T05:57:53,114 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/bc7de207d0a74b228e8c3d38ec6562b1, entries=150, sequenceid=225, filesize=30.4 K 2024-11-25T05:57:53,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/81ccbeb749354e5b99a817ec5e6375e9 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/81ccbeb749354e5b99a817ec5e6375e9 2024-11-25T05:57:53,119 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/81ccbeb749354e5b99a817ec5e6375e9, entries=150, sequenceid=225, filesize=11.9 K 2024-11-25T05:57:53,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/a4bd4cb3470f40319d8353509cc5c9c0 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/a4bd4cb3470f40319d8353509cc5c9c0 2024-11-25T05:57:53,122 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/a4bd4cb3470f40319d8353509cc5c9c0, entries=150, sequenceid=225, filesize=11.9 K 2024-11-25T05:57:53,123 INFO [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for d756b79bb1dbffd6683a6ced0694b917 in 861ms, sequenceid=225, compaction requested=false 2024-11-25T05:57:53,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for d756b79bb1dbffd6683a6ced0694b917: 2024-11-25T05:57:53,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:53,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8ef925b832e3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-25T05:57:53,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-25T05:57:53,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-25T05:57:53,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5390 sec 2024-11-25T05:57:53,126 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 2.5430 sec 2024-11-25T05:57:53,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:53,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d756b79bb1dbffd6683a6ced0694b917 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-25T05:57:53,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=A 2024-11-25T05:57:53,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:53,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=B 2024-11-25T05:57:53,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:53,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=C 2024-11-25T05:57:53,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:53,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411253ff6e1f62e5446849ed78ace45e70bb1_d756b79bb1dbffd6683a6ced0694b917 is 50, key is test_row_0/A:col10/1732514273382/Put/seqid=0 2024-11-25T05:57:53,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742418_1594 (size=12304) 2024-11-25T05:57:53,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:53,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514333428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:53,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:53,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:53,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:53,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:53,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514333429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:53,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:53,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:53,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:53,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:53,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514333431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:53,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:53,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:53,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:53,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:53,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514333541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:53,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:53,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:53,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514333542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:53,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:53,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:53,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:53,545 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:53,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514333542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:53,545 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:53,545 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:53,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:53,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:53,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:53,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:53,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514333752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:53,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:53,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:53,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:53,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514333753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:53,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:53,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:53,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:53,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:53,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:53,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514333753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:53,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:53,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:53,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:53,786 INFO [Thread-2361 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2361. 2024-11-25T05:57:53,786 DEBUG [Thread-2361 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:57:53,786 DEBUG [Thread-2361 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:53,786 DEBUG [Thread-2361 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:53,786 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:57:53,786 INFO [Thread-2363 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2363. 2024-11-25T05:57:53,786 DEBUG [Thread-2363 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:57:53,786 DEBUG [Thread-2363 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:53,787 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:57:53,787 DEBUG [Thread-2363 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:53,789 INFO [Thread-2357 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2357. 2024-11-25T05:57:53,789 DEBUG [Thread-2357 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:57:53,789 DEBUG [Thread-2357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:53,789 DEBUG [Thread-2357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:53,789 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:57:53,790 INFO [Thread-2365 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2365. 2024-11-25T05:57:53,790 DEBUG [Thread-2365 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:57:53,790 DEBUG [Thread-2365 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:53,790 DEBUG [Thread-2365 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:53,790 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:57:53,791 INFO [Thread-2359 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2359. 2024-11-25T05:57:53,791 DEBUG [Thread-2359 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:57:53,791 DEBUG [Thread-2359 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:53,791 DEBUG [Thread-2359 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:53,791 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:57:53,794 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:57:53,797 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411253ff6e1f62e5446849ed78ace45e70bb1_d756b79bb1dbffd6683a6ced0694b917 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411253ff6e1f62e5446849ed78ace45e70bb1_d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:53,798 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/0394127c9c6040b293a269a8a01c3710, store: [table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:53,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/0394127c9c6040b293a269a8a01c3710 is 175, key is test_row_0/A:col10/1732514273382/Put/seqid=0 2024-11-25T05:57:53,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742419_1595 (size=31105) 2024-11-25T05:57:54,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:54,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:54,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514334062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:54,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514334063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:54,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:54,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514334063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:54,064 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:54,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:54,064 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:54,064 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:54,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:54,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:54,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:54,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:54,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:54,202 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=244, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/0394127c9c6040b293a269a8a01c3710 2024-11-25T05:57:54,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/d8985ef833b14930a54e94992e1f4d7c is 50, key is test_row_0/B:col10/1732514273382/Put/seqid=0 2024-11-25T05:57:54,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742420_1596 (size=12151) 2024-11-25T05:57:54,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:54,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1732514334573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:54,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:54,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T05:57:54,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57972 deadline: 1732514334574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:54,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57936 deadline: 1732514334574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 2024-11-25T05:57:54,575 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:54,575 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:54,575 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:54,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:54,575 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:54,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:54,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T05:57:54,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:54,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., hostname=8ef925b832e3,35599,1732514089005, seqNum=5 because the exception is null or not the one we care about 2024-11-25T05:57:54,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/d8985ef833b14930a54e94992e1f4d7c 2024-11-25T05:57:54,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/0d38c2e5c0b04d81869383841b7d5bc3 is 50, key is test_row_0/C:col10/1732514273382/Put/seqid=0 2024-11-25T05:57:54,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742421_1597 (size=12151) 2024-11-25T05:57:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T05:57:54,753 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:57:55,037 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/0d38c2e5c0b04d81869383841b7d5bc3 2024-11-25T05:57:55,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/0394127c9c6040b293a269a8a01c3710 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/0394127c9c6040b293a269a8a01c3710 2024-11-25T05:57:55,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/0394127c9c6040b293a269a8a01c3710, entries=150, sequenceid=244, filesize=30.4 K 2024-11-25T05:57:55,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/d8985ef833b14930a54e94992e1f4d7c as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/d8985ef833b14930a54e94992e1f4d7c 2024-11-25T05:57:55,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/d8985ef833b14930a54e94992e1f4d7c, entries=150, sequenceid=244, filesize=11.9 K 2024-11-25T05:57:55,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/0d38c2e5c0b04d81869383841b7d5bc3 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/0d38c2e5c0b04d81869383841b7d5bc3 2024-11-25T05:57:55,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/0d38c2e5c0b04d81869383841b7d5bc3, entries=150, sequenceid=244, filesize=11.9 K 2024-11-25T05:57:55,061 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for d756b79bb1dbffd6683a6ced0694b917 in 1678ms, sequenceid=244, compaction requested=true 2024-11-25T05:57:55,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d756b79bb1dbffd6683a6ced0694b917: 2024-11-25T05:57:55,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d756b79bb1dbffd6683a6ced0694b917:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T05:57:55,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:55,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d756b79bb1dbffd6683a6ced0694b917:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T05:57:55,061 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:57:55,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:55,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d756b79bb1dbffd6683a6ced0694b917:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T05:57:55,062 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:57:55,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:57:55,062 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:57:55,062 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93827 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:57:55,062 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1541): d756b79bb1dbffd6683a6ced0694b917/A is initiating minor compaction (all files) 2024-11-25T05:57:55,062 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): d756b79bb1dbffd6683a6ced0694b917/B is initiating minor compaction (all files) 2024-11-25T05:57:55,062 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d756b79bb1dbffd6683a6ced0694b917/A in TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:55,062 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d756b79bb1dbffd6683a6ced0694b917/B in TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:55,063 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/402f6017e2804d74b7f57fb36f6db873, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/81ccbeb749354e5b99a817ec5e6375e9, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/d8985ef833b14930a54e94992e1f4d7c] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp, totalSize=36.1 K 2024-11-25T05:57:55,063 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/c970d85ef3f14aa4b0db34a404697685, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/bc7de207d0a74b228e8c3d38ec6562b1, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/0394127c9c6040b293a269a8a01c3710] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp, totalSize=91.6 K 2024-11-25T05:57:55,063 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:55,063 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. files: [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/c970d85ef3f14aa4b0db34a404697685, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/bc7de207d0a74b228e8c3d38ec6562b1, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/0394127c9c6040b293a269a8a01c3710] 2024-11-25T05:57:55,063 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 402f6017e2804d74b7f57fb36f6db873, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732514269408 2024-11-25T05:57:55,063 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting c970d85ef3f14aa4b0db34a404697685, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732514269408 2024-11-25T05:57:55,063 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 81ccbeb749354e5b99a817ec5e6375e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732514270565 2024-11-25T05:57:55,063 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc7de207d0a74b228e8c3d38ec6562b1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732514270565 2024-11-25T05:57:55,064 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting d8985ef833b14930a54e94992e1f4d7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1732514272741 2024-11-25T05:57:55,064 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0394127c9c6040b293a269a8a01c3710, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1732514272741 2024-11-25T05:57:55,068 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:55,069 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d756b79bb1dbffd6683a6ced0694b917#B#compaction#506 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:55,069 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125089b7f710bbc48d79e15dc1cfaf7ec99_d756b79bb1dbffd6683a6ced0694b917 store=[table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:55,069 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/25916fe76c7443ae93902bc985c7b9ae is 50, key is test_row_0/B:col10/1732514273382/Put/seqid=0 2024-11-25T05:57:55,071 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125089b7f710bbc48d79e15dc1cfaf7ec99_d756b79bb1dbffd6683a6ced0694b917, store=[table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:55,072 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125089b7f710bbc48d79e15dc1cfaf7ec99_d756b79bb1dbffd6683a6ced0694b917 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:55,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742422_1598 (size=12765) 2024-11-25T05:57:55,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742423_1599 (size=4469) 2024-11-25T05:57:55,478 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d756b79bb1dbffd6683a6ced0694b917#A#compaction#507 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:55,479 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/26394923b80f4096ac698acd92858442 is 175, key is test_row_0/A:col10/1732514273382/Put/seqid=0 2024-11-25T05:57:55,483 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/25916fe76c7443ae93902bc985c7b9ae as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/25916fe76c7443ae93902bc985c7b9ae 2024-11-25T05:57:55,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742424_1600 (size=31719) 2024-11-25T05:57:55,488 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d756b79bb1dbffd6683a6ced0694b917/B of d756b79bb1dbffd6683a6ced0694b917 into 25916fe76c7443ae93902bc985c7b9ae(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:55,488 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d756b79bb1dbffd6683a6ced0694b917: 2024-11-25T05:57:55,488 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., storeName=d756b79bb1dbffd6683a6ced0694b917/B, priority=13, startTime=1732514275061; duration=0sec 2024-11-25T05:57:55,488 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T05:57:55,488 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d756b79bb1dbffd6683a6ced0694b917:B 2024-11-25T05:57:55,488 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T05:57:55,489 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T05:57:55,489 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1541): d756b79bb1dbffd6683a6ced0694b917/C is initiating minor compaction (all files) 2024-11-25T05:57:55,489 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d756b79bb1dbffd6683a6ced0694b917/C in TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:57:55,489 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/6371173559d342ac845efcaa5d001b55, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/a4bd4cb3470f40319d8353509cc5c9c0, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/0d38c2e5c0b04d81869383841b7d5bc3] into tmpdir=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp, totalSize=36.1 K 2024-11-25T05:57:55,489 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 6371173559d342ac845efcaa5d001b55, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732514269408 2024-11-25T05:57:55,490 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting a4bd4cb3470f40319d8353509cc5c9c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732514270565 2024-11-25T05:57:55,490 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d38c2e5c0b04d81869383841b7d5bc3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1732514272741 2024-11-25T05:57:55,496 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d756b79bb1dbffd6683a6ced0694b917#C#compaction#508 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T05:57:55,496 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/549e1c37721f4ed280864daf797a4974 is 50, key is test_row_0/C:col10/1732514273382/Put/seqid=0 2024-11-25T05:57:55,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742425_1601 (size=12765) 2024-11-25T05:57:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35599 {}] regionserver.HRegion(8855): Flush requested on d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:55,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d756b79bb1dbffd6683a6ced0694b917 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-25T05:57:55,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=A 2024-11-25T05:57:55,585 INFO [Thread-2352 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2352. 2024-11-25T05:57:55,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:55,585 DEBUG [Thread-2352 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:57:55,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=B 2024-11-25T05:57:55,585 DEBUG [Thread-2352 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:55,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:55,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=C 2024-11-25T05:57:55,586 DEBUG [Thread-2352 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:55,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:57:55,586 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:57:55,595 INFO [Thread-2346 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2346. 2024-11-25T05:57:55,595 INFO [Thread-2348 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2348. 2024-11-25T05:57:55,595 DEBUG [Thread-2346 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:57:55,595 DEBUG [Thread-2348 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:57:55,595 DEBUG [Thread-2346 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:55,595 DEBUG [Thread-2348 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:55,595 DEBUG [Thread-2346 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:55,595 DEBUG [Thread-2348 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:57:55,595 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:57:55,595 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:57:55,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125509455519b2640169cc4e66e69e69787_d756b79bb1dbffd6683a6ced0694b917 is 50, key is test_row_0/A:col10/1732514275583/Put/seqid=0 2024-11-25T05:57:55,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742426_1602 (size=12404) 2024-11-25T05:57:55,894 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/26394923b80f4096ac698acd92858442 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/26394923b80f4096ac698acd92858442 2024-11-25T05:57:55,898 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d756b79bb1dbffd6683a6ced0694b917/A of d756b79bb1dbffd6683a6ced0694b917 into 26394923b80f4096ac698acd92858442(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:55,898 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d756b79bb1dbffd6683a6ced0694b917: 2024-11-25T05:57:55,898 INFO [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., storeName=d756b79bb1dbffd6683a6ced0694b917/A, priority=13, startTime=1732514275061; duration=0sec 2024-11-25T05:57:55,898 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:55,898 DEBUG [RS:0;8ef925b832e3:35599-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d756b79bb1dbffd6683a6ced0694b917:A 2024-11-25T05:57:55,903 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/549e1c37721f4ed280864daf797a4974 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/549e1c37721f4ed280864daf797a4974 2024-11-25T05:57:55,906 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d756b79bb1dbffd6683a6ced0694b917/C of d756b79bb1dbffd6683a6ced0694b917 into 549e1c37721f4ed280864daf797a4974(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T05:57:55,906 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d756b79bb1dbffd6683a6ced0694b917: 2024-11-25T05:57:55,906 INFO [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., storeName=d756b79bb1dbffd6683a6ced0694b917/C, priority=13, startTime=1732514275061; duration=0sec 2024-11-25T05:57:55,906 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T05:57:55,906 DEBUG [RS:0;8ef925b832e3:35599-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d756b79bb1dbffd6683a6ced0694b917:C 2024-11-25T05:57:56,000 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:57:56,004 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125509455519b2640169cc4e66e69e69787_d756b79bb1dbffd6683a6ced0694b917 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125509455519b2640169cc4e66e69e69787_d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:57:56,005 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/773dddb68c104b82bbcebd5e2aa2b5d6, store: [table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:57:56,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/773dddb68c104b82bbcebd5e2aa2b5d6 is 175, key is test_row_0/A:col10/1732514275583/Put/seqid=0 2024-11-25T05:57:56,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742427_1603 (size=31205) 2024-11-25T05:57:56,010 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=262, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/773dddb68c104b82bbcebd5e2aa2b5d6 2024-11-25T05:57:56,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/d4076e339523460992e5c600d9f14621 is 50, key is test_row_0/B:col10/1732514275583/Put/seqid=0 2024-11-25T05:57:56,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742428_1604 (size=12251) 2024-11-25T05:57:56,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/d4076e339523460992e5c600d9f14621 2024-11-25T05:57:56,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/a37eb090b88445879b44465a8441a7b1 is 50, key is test_row_0/C:col10/1732514275583/Put/seqid=0 2024-11-25T05:57:56,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742429_1605 (size=12251) 2024-11-25T05:57:56,844 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/a37eb090b88445879b44465a8441a7b1 2024-11-25T05:57:56,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/773dddb68c104b82bbcebd5e2aa2b5d6 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/773dddb68c104b82bbcebd5e2aa2b5d6 2024-11-25T05:57:56,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/773dddb68c104b82bbcebd5e2aa2b5d6, entries=150, sequenceid=262, filesize=30.5 K 2024-11-25T05:57:56,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/d4076e339523460992e5c600d9f14621 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/d4076e339523460992e5c600d9f14621 2024-11-25T05:57:56,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/d4076e339523460992e5c600d9f14621, entries=150, sequenceid=262, filesize=12.0 K 2024-11-25T05:57:56,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/a37eb090b88445879b44465a8441a7b1 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/a37eb090b88445879b44465a8441a7b1 2024-11-25T05:57:56,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/a37eb090b88445879b44465a8441a7b1, entries=150, sequenceid=262, filesize=12.0 K 2024-11-25T05:57:56,867 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=13.42 KB/13740 for d756b79bb1dbffd6683a6ced0694b917 in 1282ms, sequenceid=262, compaction requested=false 2024-11-25T05:57:56,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d756b79bb1dbffd6683a6ced0694b917: 2024-11-25T05:58:02,176 INFO [Thread-2350 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2350. 2024-11-25T05:58:02,176 DEBUG [Thread-2350 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:58:02,177 DEBUG [Thread-2350 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:58:02,177 DEBUG [Thread-2350 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:58:02,177 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:58:02,234 INFO [Thread-2354 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2354. 2024-11-25T05:58:02,234 DEBUG [Thread-2354 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T05:58:02,234 DEBUG [Thread-2354 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:58:02,234 DEBUG [Thread-2354 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:58:02,234 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:58:02,234 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-25T05:58:02,234 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-25T05:58:02,234 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-25T05:58:02,234 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-11-25T05:58:02,234 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-25T05:58:02,234 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-11-25T05:58:02,234 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-25T05:58:02,234 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3528 2024-11-25T05:58:02,234 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3364 2024-11-25T05:58:02,234 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3367 2024-11-25T05:58:02,234 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3526 2024-11-25T05:58:02,234 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3373 2024-11-25T05:58:02,234 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-25T05:58:02,234 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T05:58:02,234 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T05:58:02,234 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:58:02,235 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:58:02,235 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:58:02,235 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-25T05:58:02,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-25T05:58:02,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-25T05:58:02,237 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732514282237"}]},"ts":"1732514282237"} 2024-11-25T05:58:02,238 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-25T05:58:02,238 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-25T05:58:02,239 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T05:58:02,240 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d756b79bb1dbffd6683a6ced0694b917, UNASSIGN}] 2024-11-25T05:58:02,241 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=152, ppid=151, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d756b79bb1dbffd6683a6ced0694b917, UNASSIGN 2024-11-25T05:58:02,242 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=152 updating hbase:meta row=d756b79bb1dbffd6683a6ced0694b917, regionState=CLOSING, regionLocation=8ef925b832e3,35599,1732514089005 2024-11-25T05:58:02,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d756b79bb1dbffd6683a6ced0694b917, UNASSIGN because future has completed 2024-11-25T05:58:02,243 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T05:58:02,243 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; CloseRegionProcedure d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005}] 2024-11-25T05:58:02,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-25T05:58:02,396 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] handler.UnassignRegionHandler(122): Close d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:58:02,396 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T05:58:02,396 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1722): Closing d756b79bb1dbffd6683a6ced0694b917, disabling compactions & flushes 2024-11-25T05:58:02,396 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:58:02,396 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:58:02,396 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. after waiting 0 ms 2024-11-25T05:58:02,396 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:58:02,396 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(2902): Flushing d756b79bb1dbffd6683a6ced0694b917 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-25T05:58:02,397 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=A 2024-11-25T05:58:02,397 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:58:02,397 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=B 2024-11-25T05:58:02,397 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:58:02,397 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d756b79bb1dbffd6683a6ced0694b917, store=C 2024-11-25T05:58:02,397 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-25T05:58:02,405 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411252864d9f905ae41f28981c49b95b98553_d756b79bb1dbffd6683a6ced0694b917 is 50, key is test_row_1/A:col10/1732514275592/Put/seqid=0 2024-11-25T05:58:02,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742430_1606 (size=9914) 2024-11-25T05:58:02,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-25T05:58:02,809 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T05:58:02,816 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411252864d9f905ae41f28981c49b95b98553_d756b79bb1dbffd6683a6ced0694b917 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252864d9f905ae41f28981c49b95b98553_d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:58:02,818 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/feb58813bd7a43c1a10acf1323d05202, store: [table=TestAcidGuarantees family=A region=d756b79bb1dbffd6683a6ced0694b917] 2024-11-25T05:58:02,819 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/feb58813bd7a43c1a10acf1323d05202 is 175, key is test_row_1/A:col10/1732514275592/Put/seqid=0 2024-11-25T05:58:02,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742431_1607 (size=22561) 2024-11-25T05:58:02,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-25T05:58:03,225 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=271, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/feb58813bd7a43c1a10acf1323d05202 2024-11-25T05:58:03,236 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/d349e3e6673a40e5b43fc9ad75eed3db is 50, key is test_row_1/B:col10/1732514275592/Put/seqid=0 2024-11-25T05:58:03,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742432_1608 (size=9857) 2024-11-25T05:58:03,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-25T05:58:03,640 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/d349e3e6673a40e5b43fc9ad75eed3db 2024-11-25T05:58:03,645 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/acf97b88f47b4b8eaee294ea5f516031 is 50, key is test_row_1/C:col10/1732514275592/Put/seqid=0 2024-11-25T05:58:03,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742433_1609 (size=9857) 2024-11-25T05:58:04,049 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/acf97b88f47b4b8eaee294ea5f516031 2024-11-25T05:58:04,057 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/A/feb58813bd7a43c1a10acf1323d05202 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/feb58813bd7a43c1a10acf1323d05202 2024-11-25T05:58:04,060 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/feb58813bd7a43c1a10acf1323d05202, entries=100, sequenceid=271, filesize=22.0 K 2024-11-25T05:58:04,060 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/B/d349e3e6673a40e5b43fc9ad75eed3db as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/d349e3e6673a40e5b43fc9ad75eed3db 2024-11-25T05:58:04,063 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/d349e3e6673a40e5b43fc9ad75eed3db, entries=100, sequenceid=271, filesize=9.6 K 2024-11-25T05:58:04,064 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/.tmp/C/acf97b88f47b4b8eaee294ea5f516031 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/acf97b88f47b4b8eaee294ea5f516031 2024-11-25T05:58:04,066 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/acf97b88f47b4b8eaee294ea5f516031, entries=100, sequenceid=271, filesize=9.6 K 2024-11-25T05:58:04,067 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for d756b79bb1dbffd6683a6ced0694b917 in 1671ms, sequenceid=271, compaction requested=true 2024-11-25T05:58:04,067 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/55e9f73e23f14d04aeceba53191115b4, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/6af42a61390d47f88235c05722d79469, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/b2c691d7963747079d353bdf91c35f7a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/2604ea92049942cd8a5d94b665acaf42, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/b227b28b153841f2b7d8e66511037cb3, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/97e4b98ea62045d5a0d8e3458996cbc1, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/c222fdb5749d47cfa45fb351c787ad9c, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/761c348de1494430ae65647fe1435eea, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/03f9c20376a34388aab80964105bb31f, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/88f4507e1d804a0caff3fae07a86de77, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/5da9fb984da44bb99cfe501a62c74e1a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/672c8f3b2d1a4ab5ac0b7631c0bf67c7, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/e7176885638445808b2f06cd3c6c52b4, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/31781db0e2574399832e47fbfc830976, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/1f53283c5a564f5db7f0889ebf60f8bd, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/c970d85ef3f14aa4b0db34a404697685, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/bc7de207d0a74b228e8c3d38ec6562b1, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/0394127c9c6040b293a269a8a01c3710] to archive 2024-11-25T05:58:04,068 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T05:58:04,069 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/55e9f73e23f14d04aeceba53191115b4 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/55e9f73e23f14d04aeceba53191115b4 2024-11-25T05:58:04,070 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/6af42a61390d47f88235c05722d79469 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/6af42a61390d47f88235c05722d79469 2024-11-25T05:58:04,071 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/b2c691d7963747079d353bdf91c35f7a to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/b2c691d7963747079d353bdf91c35f7a 2024-11-25T05:58:04,072 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/2604ea92049942cd8a5d94b665acaf42 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/2604ea92049942cd8a5d94b665acaf42 2024-11-25T05:58:04,072 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/b227b28b153841f2b7d8e66511037cb3 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/b227b28b153841f2b7d8e66511037cb3 2024-11-25T05:58:04,073 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/97e4b98ea62045d5a0d8e3458996cbc1 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/97e4b98ea62045d5a0d8e3458996cbc1 2024-11-25T05:58:04,074 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/c222fdb5749d47cfa45fb351c787ad9c to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/c222fdb5749d47cfa45fb351c787ad9c 2024-11-25T05:58:04,074 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/761c348de1494430ae65647fe1435eea to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/761c348de1494430ae65647fe1435eea 2024-11-25T05:58:04,075 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/03f9c20376a34388aab80964105bb31f to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/03f9c20376a34388aab80964105bb31f 2024-11-25T05:58:04,076 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/88f4507e1d804a0caff3fae07a86de77 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/88f4507e1d804a0caff3fae07a86de77 2024-11-25T05:58:04,076 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/5da9fb984da44bb99cfe501a62c74e1a to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/5da9fb984da44bb99cfe501a62c74e1a 2024-11-25T05:58:04,077 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/672c8f3b2d1a4ab5ac0b7631c0bf67c7 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/672c8f3b2d1a4ab5ac0b7631c0bf67c7 2024-11-25T05:58:04,078 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/e7176885638445808b2f06cd3c6c52b4 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/e7176885638445808b2f06cd3c6c52b4 2024-11-25T05:58:04,079 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/31781db0e2574399832e47fbfc830976 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/31781db0e2574399832e47fbfc830976 2024-11-25T05:58:04,079 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/1f53283c5a564f5db7f0889ebf60f8bd to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/1f53283c5a564f5db7f0889ebf60f8bd 2024-11-25T05:58:04,080 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/c970d85ef3f14aa4b0db34a404697685 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/c970d85ef3f14aa4b0db34a404697685 2024-11-25T05:58:04,081 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/bc7de207d0a74b228e8c3d38ec6562b1 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/bc7de207d0a74b228e8c3d38ec6562b1 2024-11-25T05:58:04,082 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/0394127c9c6040b293a269a8a01c3710 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/0394127c9c6040b293a269a8a01c3710 2024-11-25T05:58:04,083 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/a0a15d55e1dd48c9b9542291a0619095, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/690c65d9842645c9ab83a045c1e592e3, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/f775ce5056b24c0497cf37bbd9ed9951, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/8a3c4847538a41d389f3bd8ebb35996b, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/c40046aee2974ff3bd14fee1e595b945, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/5e96d5fe5ab44b57b227aec97af15802, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/dd0a16b028da43378ef1c338e752124e, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/f6c3b60d148f4e9a80b834f807c070c2, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/bbb59c03a12042dcb0538acce08ff093, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/799423c5c81649ac8b9edf9a299c6f12, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/f9e66aacdd7d4b9f9a175a4c98173fb9, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/9ed00792d9d04a5e8aa63d593321debe, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/3f76e5b216b041fab7883897e14f7a24, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/332f982340f94a1cba2a5be0d75a1181, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/402f6017e2804d74b7f57fb36f6db873, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/8e2df617a9084035af3502c7f2e85e94, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/81ccbeb749354e5b99a817ec5e6375e9, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/d8985ef833b14930a54e94992e1f4d7c] to archive 2024-11-25T05:58:04,084 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T05:58:04,085 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/a0a15d55e1dd48c9b9542291a0619095 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/a0a15d55e1dd48c9b9542291a0619095 2024-11-25T05:58:04,086 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/690c65d9842645c9ab83a045c1e592e3 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/690c65d9842645c9ab83a045c1e592e3 2024-11-25T05:58:04,087 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/f775ce5056b24c0497cf37bbd9ed9951 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/f775ce5056b24c0497cf37bbd9ed9951 2024-11-25T05:58:04,088 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/8a3c4847538a41d389f3bd8ebb35996b to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/8a3c4847538a41d389f3bd8ebb35996b 2024-11-25T05:58:04,089 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/c40046aee2974ff3bd14fee1e595b945 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/c40046aee2974ff3bd14fee1e595b945 2024-11-25T05:58:04,090 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/5e96d5fe5ab44b57b227aec97af15802 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/5e96d5fe5ab44b57b227aec97af15802 2024-11-25T05:58:04,091 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/dd0a16b028da43378ef1c338e752124e to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/dd0a16b028da43378ef1c338e752124e 2024-11-25T05:58:04,092 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/f6c3b60d148f4e9a80b834f807c070c2 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/f6c3b60d148f4e9a80b834f807c070c2 2024-11-25T05:58:04,093 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/bbb59c03a12042dcb0538acce08ff093 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/bbb59c03a12042dcb0538acce08ff093 2024-11-25T05:58:04,094 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/799423c5c81649ac8b9edf9a299c6f12 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/799423c5c81649ac8b9edf9a299c6f12 2024-11-25T05:58:04,094 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/f9e66aacdd7d4b9f9a175a4c98173fb9 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/f9e66aacdd7d4b9f9a175a4c98173fb9 2024-11-25T05:58:04,095 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/9ed00792d9d04a5e8aa63d593321debe to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/9ed00792d9d04a5e8aa63d593321debe 2024-11-25T05:58:04,095 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/3f76e5b216b041fab7883897e14f7a24 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/3f76e5b216b041fab7883897e14f7a24 2024-11-25T05:58:04,096 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/332f982340f94a1cba2a5be0d75a1181 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/332f982340f94a1cba2a5be0d75a1181 2024-11-25T05:58:04,097 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/402f6017e2804d74b7f57fb36f6db873 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/402f6017e2804d74b7f57fb36f6db873 2024-11-25T05:58:04,098 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/8e2df617a9084035af3502c7f2e85e94 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/8e2df617a9084035af3502c7f2e85e94 2024-11-25T05:58:04,098 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/81ccbeb749354e5b99a817ec5e6375e9 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/81ccbeb749354e5b99a817ec5e6375e9 2024-11-25T05:58:04,099 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/d8985ef833b14930a54e94992e1f4d7c to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/d8985ef833b14930a54e94992e1f4d7c 2024-11-25T05:58:04,100 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/5c54f273e3494abfb92133b699696006, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/e1426950801143e0beb1548d61efdcbb, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/d8a6709cf3e04140b9c9d8be80405073, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/4deef89ffeef4983a817efd7bccc3e3a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/8e9f3bc74bca48709781ea0e4bdb5840, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/465eceb14e434c7197f1ee57e8c61ca4, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/ae96ae6839974d42b450eba59ee64697, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/308e1226aee745dca3a80bf8ff21405d, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/67e0d12f91e8447c97888974adf5e696, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/5d1149ce0e944567b8533d9af67ebe6a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/51645033018f427eb8e60ded5443229e, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/c7e0580c9d344c00aa826f0c1bf8b622, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/e1600ebc2a7748f8990657888c0d8060, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/bcdb4e89e7604e1896e22a49bb56f3a5, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/6371173559d342ac845efcaa5d001b55, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/748d67be63e54be589bffd2b00df8a3a, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/a4bd4cb3470f40319d8353509cc5c9c0, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/0d38c2e5c0b04d81869383841b7d5bc3] to archive 2024-11-25T05:58:04,100 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T05:58:04,101 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/5c54f273e3494abfb92133b699696006 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/5c54f273e3494abfb92133b699696006 2024-11-25T05:58:04,102 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/e1426950801143e0beb1548d61efdcbb to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/e1426950801143e0beb1548d61efdcbb 2024-11-25T05:58:04,103 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/d8a6709cf3e04140b9c9d8be80405073 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/d8a6709cf3e04140b9c9d8be80405073 2024-11-25T05:58:04,103 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/4deef89ffeef4983a817efd7bccc3e3a to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/4deef89ffeef4983a817efd7bccc3e3a 2024-11-25T05:58:04,104 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/8e9f3bc74bca48709781ea0e4bdb5840 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/8e9f3bc74bca48709781ea0e4bdb5840 2024-11-25T05:58:04,104 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/465eceb14e434c7197f1ee57e8c61ca4 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/465eceb14e434c7197f1ee57e8c61ca4 2024-11-25T05:58:04,105 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/ae96ae6839974d42b450eba59ee64697 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/ae96ae6839974d42b450eba59ee64697 2024-11-25T05:58:04,106 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/308e1226aee745dca3a80bf8ff21405d to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/308e1226aee745dca3a80bf8ff21405d 2024-11-25T05:58:04,107 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/67e0d12f91e8447c97888974adf5e696 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/67e0d12f91e8447c97888974adf5e696 2024-11-25T05:58:04,107 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/5d1149ce0e944567b8533d9af67ebe6a to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/5d1149ce0e944567b8533d9af67ebe6a 2024-11-25T05:58:04,108 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/51645033018f427eb8e60ded5443229e to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/51645033018f427eb8e60ded5443229e 2024-11-25T05:58:04,108 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/c7e0580c9d344c00aa826f0c1bf8b622 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/c7e0580c9d344c00aa826f0c1bf8b622 2024-11-25T05:58:04,109 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/e1600ebc2a7748f8990657888c0d8060 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/e1600ebc2a7748f8990657888c0d8060 2024-11-25T05:58:04,110 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/bcdb4e89e7604e1896e22a49bb56f3a5 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/bcdb4e89e7604e1896e22a49bb56f3a5 2024-11-25T05:58:04,110 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/6371173559d342ac845efcaa5d001b55 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/6371173559d342ac845efcaa5d001b55 2024-11-25T05:58:04,111 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/748d67be63e54be589bffd2b00df8a3a to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/748d67be63e54be589bffd2b00df8a3a 2024-11-25T05:58:04,112 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/a4bd4cb3470f40319d8353509cc5c9c0 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/a4bd4cb3470f40319d8353509cc5c9c0 2024-11-25T05:58:04,112 DEBUG [StoreCloser-TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/0d38c2e5c0b04d81869383841b7d5bc3 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/0d38c2e5c0b04d81869383841b7d5bc3 2024-11-25T05:58:04,115 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/recovered.edits/274.seqid, newMaxSeqId=274, maxSeqId=4 2024-11-25T05:58:04,116 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. 2024-11-25T05:58:04,116 DEBUG [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1676): Region close journal for d756b79bb1dbffd6683a6ced0694b917: Waiting for close lock at 1732514282396Running coprocessor pre-close hooks at 1732514282396Disabling compacts and flushes for region at 1732514282396Disabling writes for close at 1732514282396Obtaining lock to block concurrent updates at 1732514282396Preparing flush snapshotting stores in d756b79bb1dbffd6683a6ced0694b917 at 1732514282396Finished memstore snapshotting TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732514282397 (+1 ms)Flushing stores of TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917. at 1732514282398 (+1 ms)Flushing d756b79bb1dbffd6683a6ced0694b917/A: creating writer at 1732514282398Flushing mob file d756b79bb1dbffd6683a6ced0694b917/A: appending metadata at 1732514282404 (+6 ms)Flushing mob file d756b79bb1dbffd6683a6ced0694b917/A: closing flushed file at 1732514282404Flushing d756b79bb1dbffd6683a6ced0694b917/A: appending metadata at 1732514282818 (+414 ms)Flushing d756b79bb1dbffd6683a6ced0694b917/A: closing flushed file at 1732514282818Flushing d756b79bb1dbffd6683a6ced0694b917/B: creating writer at 1732514283231 (+413 ms)Flushing d756b79bb1dbffd6683a6ced0694b917/B: appending metadata at 1732514283236 (+5 ms)Flushing d756b79bb1dbffd6683a6ced0694b917/B: closing flushed file at 1732514283236Flushing d756b79bb1dbffd6683a6ced0694b917/C: creating writer at 1732514283643 (+407 ms)Flushing d756b79bb1dbffd6683a6ced0694b917/C: appending metadata at 1732514283645 (+2 ms)Flushing d756b79bb1dbffd6683a6ced0694b917/C: closing flushed file at 1732514283645Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ede94ad: reopening flushed file at 1732514284057 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b5974e2: reopening flushed file at 1732514284060 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@359ea6ad: reopening flushed file at 1732514284063 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for d756b79bb1dbffd6683a6ced0694b917 in 1671ms, sequenceid=271, compaction requested=true at 1732514284067 (+4 ms)Writing region close event to WAL at 1732514284113 (+46 ms)Running coprocessor post-close hooks at 1732514284116 (+3 ms)Closed at 1732514284116 2024-11-25T05:58:04,117 INFO [RS_CLOSE_REGION-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] handler.UnassignRegionHandler(157): Closed d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:58:04,118 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=152 updating hbase:meta row=d756b79bb1dbffd6683a6ced0694b917, regionState=CLOSED 2024-11-25T05:58:04,119 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE, hasLock=false; CloseRegionProcedure d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 because future has completed 2024-11-25T05:58:04,120 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-25T05:58:04,120 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; CloseRegionProcedure d756b79bb1dbffd6683a6ced0694b917, server=8ef925b832e3,35599,1732514089005 in 1.8760 sec 2024-11-25T05:58:04,121 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-25T05:58:04,121 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d756b79bb1dbffd6683a6ced0694b917, UNASSIGN in 1.8800 sec 2024-11-25T05:58:04,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-25T05:58:04,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8830 sec 2024-11-25T05:58:04,124 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732514284124"}]},"ts":"1732514284124"} 2024-11-25T05:58:04,125 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-25T05:58:04,125 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-25T05:58:04,126 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8900 sec 2024-11-25T05:58:04,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-25T05:58:04,383 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:58:04,385 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-25T05:58:04,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T05:58:04,389 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=154, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T05:58:04,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-25T05:58:04,391 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=154, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T05:58:04,396 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:58:04,399 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A, FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B, FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C, FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/recovered.edits] 2024-11-25T05:58:04,403 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/26394923b80f4096ac698acd92858442 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/26394923b80f4096ac698acd92858442 2024-11-25T05:58:04,405 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/773dddb68c104b82bbcebd5e2aa2b5d6 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/773dddb68c104b82bbcebd5e2aa2b5d6 2024-11-25T05:58:04,406 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/feb58813bd7a43c1a10acf1323d05202 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/A/feb58813bd7a43c1a10acf1323d05202 2024-11-25T05:58:04,408 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/25916fe76c7443ae93902bc985c7b9ae to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/25916fe76c7443ae93902bc985c7b9ae 2024-11-25T05:58:04,410 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/d349e3e6673a40e5b43fc9ad75eed3db to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/d349e3e6673a40e5b43fc9ad75eed3db 2024-11-25T05:58:04,411 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/d4076e339523460992e5c600d9f14621 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/B/d4076e339523460992e5c600d9f14621 2024-11-25T05:58:04,414 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/549e1c37721f4ed280864daf797a4974 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/549e1c37721f4ed280864daf797a4974 2024-11-25T05:58:04,415 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/a37eb090b88445879b44465a8441a7b1 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/a37eb090b88445879b44465a8441a7b1 2024-11-25T05:58:04,417 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/acf97b88f47b4b8eaee294ea5f516031 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/C/acf97b88f47b4b8eaee294ea5f516031 2024-11-25T05:58:04,421 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/recovered.edits/274.seqid to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917/recovered.edits/274.seqid 2024-11-25T05:58:04,422 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/default/TestAcidGuarantees/d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:58:04,422 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-25T05:58:04,423 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-25T05:58:04,424 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-25T05:58:04,428 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250ee2d1c3b7c04351b87317513f12396f_d756b79bb1dbffd6683a6ced0694b917 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250ee2d1c3b7c04351b87317513f12396f_d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:58:04,430 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252864d9f905ae41f28981c49b95b98553_d756b79bb1dbffd6683a6ced0694b917 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252864d9f905ae41f28981c49b95b98553_d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:58:04,432 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112539a9a88387fe405f82439c1ec6cab177_d756b79bb1dbffd6683a6ced0694b917 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112539a9a88387fe405f82439c1ec6cab177_d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:58:04,434 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411253ff6e1f62e5446849ed78ace45e70bb1_d756b79bb1dbffd6683a6ced0694b917 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411253ff6e1f62e5446849ed78ace45e70bb1_d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:58:04,436 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112547bac1310c0842d1ba1f6f1fc2ca2398_d756b79bb1dbffd6683a6ced0694b917 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112547bac1310c0842d1ba1f6f1fc2ca2398_d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:58:04,438 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125509455519b2640169cc4e66e69e69787_d756b79bb1dbffd6683a6ced0694b917 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125509455519b2640169cc4e66e69e69787_d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:58:04,439 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256cf2ff83287146b886d0ea16014dfd24_d756b79bb1dbffd6683a6ced0694b917 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256cf2ff83287146b886d0ea16014dfd24_d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:58:04,441 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411257ac16e1802f1416881b5a1a001ee687e_d756b79bb1dbffd6683a6ced0694b917 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411257ac16e1802f1416881b5a1a001ee687e_d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:58:04,442 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411257c996edb56144cc2a321119099430f36_d756b79bb1dbffd6683a6ced0694b917 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411257c996edb56144cc2a321119099430f36_d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:58:04,443 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112581f29ef5208343129d3abe5fc7a5c683_d756b79bb1dbffd6683a6ced0694b917 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112581f29ef5208343129d3abe5fc7a5c683_d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:58:04,445 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125872701eecaf34caaa07303800814c76d_d756b79bb1dbffd6683a6ced0694b917 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125872701eecaf34caaa07303800814c76d_d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:58:04,447 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125bb206d45b1644e89baa88500a697a7c4_d756b79bb1dbffd6683a6ced0694b917 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125bb206d45b1644e89baa88500a697a7c4_d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:58:04,448 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c1d8ce4733ef44fea1ebcf348b3067ea_d756b79bb1dbffd6683a6ced0694b917 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c1d8ce4733ef44fea1ebcf348b3067ea_d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:58:04,450 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d22e7108cfcc4acbbb7499974d7bf9f7_d756b79bb1dbffd6683a6ced0694b917 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d22e7108cfcc4acbbb7499974d7bf9f7_d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:58:04,451 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125f5115f6cc71844398d60ef8f3d6c4624_d756b79bb1dbffd6683a6ced0694b917 to hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125f5115f6cc71844398d60ef8f3d6c4624_d756b79bb1dbffd6683a6ced0694b917 2024-11-25T05:58:04,452 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-25T05:58:04,454 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=154, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T05:58:04,456 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-25T05:58:04,458 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-25T05:58:04,458 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=154, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T05:58:04,459 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-25T05:58:04,459 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732514284459"}]},"ts":"9223372036854775807"} 2024-11-25T05:58:04,460 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-25T05:58:04,460 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d756b79bb1dbffd6683a6ced0694b917, NAME => 'TestAcidGuarantees,,1732514250639.d756b79bb1dbffd6683a6ced0694b917.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T05:58:04,460 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-25T05:58:04,460 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732514284460"}]},"ts":"9223372036854775807"} 2024-11-25T05:58:04,462 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-25T05:58:04,462 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=154, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T05:58:04,463 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 77 msec 2024-11-25T05:58:04,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-25T05:58:04,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-25T05:58:04,503 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-25T05:58:04,510 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 246) - Thread LEAK? -, OpenFileDescriptor=445 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=297 (was 293) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7566 (was 7589) 2024-11-25T05:58:04,510 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-25T05:58:04,510 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T05:58:04,510 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T05:58:04,510 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:58:04,510 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:58:04,510 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T05:58:04,510 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-25T05:58:04,510 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=716219951, stopped=false 2024-11-25T05:58:04,510 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=8ef925b832e3,41285,1732514088402 2024-11-25T05:58:04,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35599-0x10075725f200001, quorum=127.0.0.1:59687, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T05:58:04,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41285-0x10075725f200000, quorum=127.0.0.1:59687, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T05:58:04,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35599-0x10075725f200001, quorum=127.0.0.1:59687, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T05:58:04,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41285-0x10075725f200000, quorum=127.0.0.1:59687, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T05:58:04,512 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T05:58:04,513 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T05:58:04,513 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41285-0x10075725f200000, quorum=127.0.0.1:59687, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T05:58:04,513 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35599-0x10075725f200001, quorum=127.0.0.1:59687, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T05:58:04,513 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T05:58:04,513 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:58:04,513 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '8ef925b832e3,35599,1732514089005' ***** 2024-11-25T05:58:04,513 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-25T05:58:04,514 INFO [RS:0;8ef925b832e3:35599 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T05:58:04,514 INFO [RS:0;8ef925b832e3:35599 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T05:58:04,514 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T05:58:04,514 INFO [RS:0;8ef925b832e3:35599 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T05:58:04,514 INFO [RS:0;8ef925b832e3:35599 {}] regionserver.HRegionServer(959): stopping server 8ef925b832e3,35599,1732514089005 2024-11-25T05:58:04,514 INFO [RS:0;8ef925b832e3:35599 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T05:58:04,514 INFO [RS:0;8ef925b832e3:35599 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;8ef925b832e3:35599. 2024-11-25T05:58:04,514 DEBUG [RS:0;8ef925b832e3:35599 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T05:58:04,514 DEBUG [RS:0;8ef925b832e3:35599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:58:04,514 INFO [RS:0;8ef925b832e3:35599 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T05:58:04,514 INFO [RS:0;8ef925b832e3:35599 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T05:58:04,514 INFO [RS:0;8ef925b832e3:35599 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T05:58:04,515 INFO [RS:0;8ef925b832e3:35599 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-25T05:58:04,515 INFO [RS:0;8ef925b832e3:35599 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-25T05:58:04,515 DEBUG [RS:0;8ef925b832e3:35599 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-25T05:58:04,515 DEBUG [RS:0;8ef925b832e3:35599 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T05:58:04,515 DEBUG [RS_CLOSE_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T05:58:04,516 INFO [RS_CLOSE_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T05:58:04,516 DEBUG [RS_CLOSE_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T05:58:04,516 DEBUG [RS_CLOSE_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T05:58:04,516 DEBUG [RS_CLOSE_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T05:58:04,516 INFO [RS_CLOSE_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-25T05:58:04,539 DEBUG [RS_CLOSE_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/hbase/meta/1588230740/.tmp/info/fdf728e5839448ca9913a34868b5e54b is 95, key is TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34./info:/1732514120831/DeleteFamily/seqid=0 2024-11-25T05:58:04,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742434_1610 (size=6514) 2024-11-25T05:58:04,603 INFO [regionserver/8ef925b832e3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T05:58:04,701 INFO [regionserver/8ef925b832e3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-25T05:58:04,701 INFO [regionserver/8ef925b832e3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-25T05:58:04,716 DEBUG [RS:0;8ef925b832e3:35599 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T05:58:04,916 DEBUG [RS:0;8ef925b832e3:35599 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T05:58:04,943 INFO [RS_CLOSE_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/hbase/meta/1588230740/.tmp/info/fdf728e5839448ca9913a34868b5e54b 2024-11-25T05:58:04,974 DEBUG [RS_CLOSE_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/hbase/meta/1588230740/.tmp/ns/fc581c158ca340e58e28fa4157b79a2c is 93, key is TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34./ns:/1732514120784/DeleteFamily/seqid=0 2024-11-25T05:58:04,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742435_1611 (size=5906) 2024-11-25T05:58:05,116 DEBUG [RS:0;8ef925b832e3:35599 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T05:58:05,317 DEBUG [RS:0;8ef925b832e3:35599 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T05:58:05,378 INFO [RS_CLOSE_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/hbase/meta/1588230740/.tmp/ns/fc581c158ca340e58e28fa4157b79a2c 2024-11-25T05:58:05,405 DEBUG [RS_CLOSE_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/hbase/meta/1588230740/.tmp/rep_barrier/4f1642075f4c41079f5f2140d5d371d8 is 102, key is TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34./rep_barrier:/1732514120784/DeleteFamily/seqid=0 2024-11-25T05:58:05,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742436_1612 (size=6025) 2024-11-25T05:58:05,517 DEBUG [RS:0;8ef925b832e3:35599 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T05:58:05,717 DEBUG [RS:0;8ef925b832e3:35599 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T05:58:05,810 INFO [RS_CLOSE_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/hbase/meta/1588230740/.tmp/rep_barrier/4f1642075f4c41079f5f2140d5d371d8 2024-11-25T05:58:05,837 DEBUG [RS_CLOSE_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/hbase/meta/1588230740/.tmp/table/66bda0a1fadc41268bd72f05fd1479b9 is 96, key is TestAcidGuarantees,,1732514092821.cf8ce3f4f3ad5ac4b5297b0ca903cd34./table:/1732514120784/DeleteFamily/seqid=0 2024-11-25T05:58:05,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742437_1613 (size=5936) 2024-11-25T05:58:05,918 DEBUG [RS:0;8ef925b832e3:35599 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T05:58:06,118 DEBUG [RS:0;8ef925b832e3:35599 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T05:58:06,241 INFO [RS_CLOSE_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/hbase/meta/1588230740/.tmp/table/66bda0a1fadc41268bd72f05fd1479b9 2024-11-25T05:58:06,251 DEBUG [RS_CLOSE_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/hbase/meta/1588230740/.tmp/info/fdf728e5839448ca9913a34868b5e54b as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/hbase/meta/1588230740/info/fdf728e5839448ca9913a34868b5e54b 2024-11-25T05:58:06,261 INFO [RS_CLOSE_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/hbase/meta/1588230740/info/fdf728e5839448ca9913a34868b5e54b, entries=12, sequenceid=90, filesize=6.4 K 2024-11-25T05:58:06,262 DEBUG [RS_CLOSE_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/hbase/meta/1588230740/.tmp/ns/fc581c158ca340e58e28fa4157b79a2c as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/hbase/meta/1588230740/ns/fc581c158ca340e58e28fa4157b79a2c 2024-11-25T05:58:06,264 INFO [RS_CLOSE_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/hbase/meta/1588230740/ns/fc581c158ca340e58e28fa4157b79a2c, entries=8, sequenceid=90, filesize=5.8 K 2024-11-25T05:58:06,265 DEBUG [RS_CLOSE_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/hbase/meta/1588230740/.tmp/rep_barrier/4f1642075f4c41079f5f2140d5d371d8 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/hbase/meta/1588230740/rep_barrier/4f1642075f4c41079f5f2140d5d371d8 2024-11-25T05:58:06,267 INFO [RS_CLOSE_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/hbase/meta/1588230740/rep_barrier/4f1642075f4c41079f5f2140d5d371d8, entries=6, sequenceid=90, filesize=5.9 K 2024-11-25T05:58:06,268 DEBUG [RS_CLOSE_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/hbase/meta/1588230740/.tmp/table/66bda0a1fadc41268bd72f05fd1479b9 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/hbase/meta/1588230740/table/66bda0a1fadc41268bd72f05fd1479b9 2024-11-25T05:58:06,270 INFO [RS_CLOSE_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/hbase/meta/1588230740/table/66bda0a1fadc41268bd72f05fd1479b9, entries=7, sequenceid=90, filesize=5.8 K 2024-11-25T05:58:06,271 INFO [RS_CLOSE_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1755ms, sequenceid=90, compaction requested=false 2024-11-25T05:58:06,274 DEBUG [RS_CLOSE_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-25T05:58:06,275 DEBUG [RS_CLOSE_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T05:58:06,275 INFO [RS_CLOSE_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T05:58:06,275 DEBUG [RS_CLOSE_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732514284515Running coprocessor pre-close hooks at 1732514284515Disabling compacts and flushes for region at 1732514284515Disabling writes for close at 1732514284516 (+1 ms)Obtaining lock to block concurrent updates at 1732514284516Preparing flush snapshotting stores in 1588230740 at 1732514284516Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732514284516Flushing stores of hbase:meta,,1.1588230740 at 1732514284517 (+1 ms)Flushing 1588230740/info: creating writer at 1732514284517Flushing 1588230740/info: appending metadata at 1732514284539 (+22 ms)Flushing 1588230740/info: closing flushed file at 1732514284539Flushing 1588230740/ns: creating writer at 1732514284954 (+415 ms)Flushing 1588230740/ns: appending metadata at 1732514284973 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1732514284973Flushing 1588230740/rep_barrier: creating writer at 1732514285387 (+414 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732514285405 (+18 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732514285405Flushing 1588230740/table: creating writer at 1732514285820 (+415 ms)Flushing 1588230740/table: appending metadata at 1732514285836 (+16 ms)Flushing 1588230740/table: closing flushed file at 1732514285836Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23b726f3: reopening flushed file at 1732514286250 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e465692: reopening flushed file at 1732514286261 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55b1b31c: reopening flushed file at 1732514286264 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@64816be3: reopening flushed file at 1732514286268 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1755ms, sequenceid=90, compaction requested=false at 1732514286271 (+3 ms)Writing region close event to WAL at 1732514286272 (+1 ms)Running coprocessor post-close hooks at 1732514286275 (+3 ms)Closed at 1732514286275 2024-11-25T05:58:06,275 DEBUG [RS_CLOSE_META-regionserver/8ef925b832e3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-25T05:58:06,318 INFO [RS:0;8ef925b832e3:35599 {}] regionserver.HRegionServer(976): stopping server 8ef925b832e3,35599,1732514089005; all regions closed. 2024-11-25T05:58:06,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741834_1010 (size=25344) 2024-11-25T05:58:06,329 DEBUG [RS:0;8ef925b832e3:35599 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/oldWALs 2024-11-25T05:58:06,330 INFO [RS:0;8ef925b832e3:35599 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8ef925b832e3%2C35599%2C1732514089005.meta:.meta(num 1732514092406) 2024-11-25T05:58:06,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741833_1009 (size=15954182) 2024-11-25T05:58:06,336 DEBUG [RS:0;8ef925b832e3:35599 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/oldWALs 2024-11-25T05:58:06,336 INFO [RS:0;8ef925b832e3:35599 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8ef925b832e3%2C35599%2C1732514089005:(num 1732514091824) 2024-11-25T05:58:06,336 DEBUG [RS:0;8ef925b832e3:35599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T05:58:06,336 INFO [RS:0;8ef925b832e3:35599 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T05:58:06,336 INFO [RS:0;8ef925b832e3:35599 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T05:58:06,337 INFO [RS:0;8ef925b832e3:35599 {}] hbase.ChoreService(370): Chore service for: regionserver/8ef925b832e3:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-25T05:58:06,337 INFO [RS:0;8ef925b832e3:35599 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T05:58:06,337 INFO [regionserver/8ef925b832e3:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T05:58:06,338 INFO [RS:0;8ef925b832e3:35599 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35599 2024-11-25T05:58:06,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35599-0x10075725f200001, quorum=127.0.0.1:59687, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/8ef925b832e3,35599,1732514089005 2024-11-25T05:58:06,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41285-0x10075725f200000, quorum=127.0.0.1:59687, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T05:58:06,342 INFO [RS:0;8ef925b832e3:35599 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T05:58:06,343 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [8ef925b832e3,35599,1732514089005] 2024-11-25T05:58:06,344 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/8ef925b832e3,35599,1732514089005 already deleted, retry=false 2024-11-25T05:58:06,345 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 8ef925b832e3,35599,1732514089005 expired; onlineServers=0 2024-11-25T05:58:06,345 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '8ef925b832e3,41285,1732514088402' ***** 2024-11-25T05:58:06,345 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-25T05:58:06,345 INFO [M:0;8ef925b832e3:41285 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T05:58:06,345 INFO [M:0;8ef925b832e3:41285 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T05:58:06,345 DEBUG [M:0;8ef925b832e3:41285 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-25T05:58:06,345 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-25T05:58:06,345 DEBUG [M:0;8ef925b832e3:41285 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-25T05:58:06,345 DEBUG [master/8ef925b832e3:0:becomeActiveMaster-HFileCleaner.small.0-1732514091558 {}] cleaner.HFileCleaner(306): Exit Thread[master/8ef925b832e3:0:becomeActiveMaster-HFileCleaner.small.0-1732514091558,5,FailOnTimeoutGroup] 2024-11-25T05:58:06,345 DEBUG [master/8ef925b832e3:0:becomeActiveMaster-HFileCleaner.large.0-1732514091553 {}] cleaner.HFileCleaner(306): Exit Thread[master/8ef925b832e3:0:becomeActiveMaster-HFileCleaner.large.0-1732514091553,5,FailOnTimeoutGroup] 2024-11-25T05:58:06,346 INFO [M:0;8ef925b832e3:41285 {}] hbase.ChoreService(370): Chore service for: master/8ef925b832e3:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-25T05:58:06,346 INFO [M:0;8ef925b832e3:41285 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T05:58:06,346 DEBUG [M:0;8ef925b832e3:41285 {}] master.HMaster(1795): Stopping service threads 2024-11-25T05:58:06,346 INFO [M:0;8ef925b832e3:41285 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-25T05:58:06,346 INFO [M:0;8ef925b832e3:41285 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T05:58:06,346 ERROR [M:0;8ef925b832e3:41285 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:42949 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:42949,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-25T05:58:06,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41285-0x10075725f200000, quorum=127.0.0.1:59687, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-25T05:58:06,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41285-0x10075725f200000, quorum=127.0.0.1:59687, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T05:58:06,347 INFO [M:0;8ef925b832e3:41285 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T05:58:06,347 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-25T05:58:06,347 DEBUG [M:0;8ef925b832e3:41285 {}] zookeeper.ZKUtil(347): master:41285-0x10075725f200000, quorum=127.0.0.1:59687, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-25T05:58:06,347 WARN [M:0;8ef925b832e3:41285 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-25T05:58:06,348 INFO [M:0;8ef925b832e3:41285 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/.lastflushedseqids 2024-11-25T05:58:06,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742438_1614 (size=72) 2024-11-25T05:58:06,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35599-0x10075725f200001, quorum=127.0.0.1:59687, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T05:58:06,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35599-0x10075725f200001, quorum=127.0.0.1:59687, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T05:58:06,445 INFO [RS:0;8ef925b832e3:35599 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T05:58:06,446 INFO [RS:0;8ef925b832e3:35599 {}] regionserver.HRegionServer(1031): Exiting; stopping=8ef925b832e3,35599,1732514089005; zookeeper connection closed. 2024-11-25T05:58:06,446 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5ff7efe4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5ff7efe4 2024-11-25T05:58:06,447 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-25T05:58:06,758 INFO [M:0;8ef925b832e3:41285 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T05:58:06,759 INFO [M:0;8ef925b832e3:41285 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-25T05:58:06,759 DEBUG [M:0;8ef925b832e3:41285 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T05:58:06,759 INFO [M:0;8ef925b832e3:41285 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T05:58:06,759 DEBUG [M:0;8ef925b832e3:41285 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T05:58:06,759 DEBUG [M:0;8ef925b832e3:41285 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T05:58:06,759 DEBUG [M:0;8ef925b832e3:41285 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T05:58:06,760 INFO [M:0;8ef925b832e3:41285 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=741.46 KB heapSize=906.12 KB 2024-11-25T05:58:06,781 DEBUG [M:0;8ef925b832e3:41285 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/529f34b2321d4d079dc302f61d585772 is 82, key is hbase:meta,,1/info:regioninfo/1732514092485/Put/seqid=0 2024-11-25T05:58:06,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742439_1615 (size=5672) 2024-11-25T05:58:07,186 INFO [M:0;8ef925b832e3:41285 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2035 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/529f34b2321d4d079dc302f61d585772 2024-11-25T05:58:07,221 DEBUG [M:0;8ef925b832e3:41285 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/936c958122fa4097af94acd15652e799 is 2480, key is \x00\x00\x00\x00\x00\x00\x00!/proc:d/1732514124024/Put/seqid=0 2024-11-25T05:58:07,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742440_1616 (size=42633) 2024-11-25T05:58:07,626 INFO [M:0;8ef925b832e3:41285 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=740.91 KB at sequenceid=2035 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/936c958122fa4097af94acd15652e799 2024-11-25T05:58:07,636 INFO [M:0;8ef925b832e3:41285 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 936c958122fa4097af94acd15652e799 2024-11-25T05:58:07,653 DEBUG [M:0;8ef925b832e3:41285 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c1ae28806dfa49e7849546de344ba4fa is 69, key is 8ef925b832e3,35599,1732514089005/rs:state/1732514091589/Put/seqid=0 2024-11-25T05:58:07,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073742441_1617 (size=5156) 2024-11-25T05:58:08,058 INFO [M:0;8ef925b832e3:41285 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2035 (bloomFilter=true), to=hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c1ae28806dfa49e7849546de344ba4fa 2024-11-25T05:58:08,069 DEBUG [M:0;8ef925b832e3:41285 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/529f34b2321d4d079dc302f61d585772 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/529f34b2321d4d079dc302f61d585772 2024-11-25T05:58:08,075 INFO [M:0;8ef925b832e3:41285 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/529f34b2321d4d079dc302f61d585772, entries=8, sequenceid=2035, filesize=5.5 K 2024-11-25T05:58:08,076 DEBUG [M:0;8ef925b832e3:41285 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/936c958122fa4097af94acd15652e799 as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/936c958122fa4097af94acd15652e799 2024-11-25T05:58:08,081 INFO [M:0;8ef925b832e3:41285 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 936c958122fa4097af94acd15652e799 2024-11-25T05:58:08,081 INFO [M:0;8ef925b832e3:41285 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/936c958122fa4097af94acd15652e799, entries=154, sequenceid=2035, filesize=41.6 K 2024-11-25T05:58:08,082 DEBUG [M:0;8ef925b832e3:41285 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c1ae28806dfa49e7849546de344ba4fa as hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c1ae28806dfa49e7849546de344ba4fa 2024-11-25T05:58:08,086 INFO [M:0;8ef925b832e3:41285 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42949/user/jenkins/test-data/d41fcb61-7eb5-7f3b-ee77-f43220d7610a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c1ae28806dfa49e7849546de344ba4fa, entries=1, sequenceid=2035, filesize=5.0 K 2024-11-25T05:58:08,087 INFO [M:0;8ef925b832e3:41285 {}] regionserver.HRegion(3140): Finished flush of dataSize ~741.46 KB/759260, heapSize ~905.82 KB/927560, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1328ms, sequenceid=2035, compaction requested=false 2024-11-25T05:58:08,088 INFO [M:0;8ef925b832e3:41285 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T05:58:08,089 DEBUG [M:0;8ef925b832e3:41285 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732514286759Disabling compacts and flushes for region at 1732514286759Disabling writes for close at 1732514286759Obtaining lock to block concurrent updates at 1732514286760 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732514286760Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=759260, getHeapSize=927800, getOffHeapSize=0, getCellsCount=2214 at 1732514286761 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732514286762 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732514286763 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732514286781 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732514286781Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732514287194 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732514287220 (+26 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732514287220Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732514287636 (+416 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732514287653 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732514287653Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@98840e: reopening flushed file at 1732514288067 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f9bf694: reopening flushed file at 1732514288075 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@196a8b86: reopening flushed file at 1732514288081 (+6 ms)Finished flush of dataSize ~741.46 KB/759260, heapSize ~905.82 KB/927560, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1328ms, sequenceid=2035, compaction requested=false at 1732514288087 (+6 ms)Writing region close event to WAL at 1732514288088 (+1 ms)Closed at 1732514288088 2024-11-25T05:58:08,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33269 is added to blk_1073741830_1006 (size=892670) 2024-11-25T05:58:08,092 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T05:58:08,092 INFO [M:0;8ef925b832e3:41285 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-25T05:58:08,092 INFO [M:0;8ef925b832e3:41285 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41285 2024-11-25T05:58:08,093 INFO [M:0;8ef925b832e3:41285 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T05:58:08,194 INFO [M:0;8ef925b832e3:41285 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T05:58:08,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41285-0x10075725f200000, quorum=127.0.0.1:59687, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T05:58:08,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41285-0x10075725f200000, quorum=127.0.0.1:59687, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T05:58:08,198 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T05:58:08,200 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T05:58:08,200 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T05:58:08,200 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-25T05:58:08,201 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6586672e-597e-41c1-14d9-2bcb3b602f07/hadoop.log.dir/,STOPPED} 2024-11-25T05:58:08,203 WARN [BP-1069327184-172.17.0.2-1732514085807 heartbeating to localhost/127.0.0.1:42949 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-25T05:58:08,203 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-25T05:58:08,203 WARN [BP-1069327184-172.17.0.2-1732514085807 heartbeating to localhost/127.0.0.1:42949 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1069327184-172.17.0.2-1732514085807 (Datanode Uuid 722df91b-1e78-4820-8141-ffa7be2e3ce0) service to localhost/127.0.0.1:42949 2024-11-25T05:58:08,203 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-25T05:58:08,205 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6586672e-597e-41c1-14d9-2bcb3b602f07/cluster_8af6f6a5-ddee-228b-0d29-2ed39e53a991/data/data1/current/BP-1069327184-172.17.0.2-1732514085807 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T05:58:08,205 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6586672e-597e-41c1-14d9-2bcb3b602f07/cluster_8af6f6a5-ddee-228b-0d29-2ed39e53a991/data/data2/current/BP-1069327184-172.17.0.2-1732514085807 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T05:58:08,206 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-25T05:58:08,213 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-25T05:58:08,213 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T05:58:08,213 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T05:58:08,214 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-25T05:58:08,214 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6586672e-597e-41c1-14d9-2bcb3b602f07/hadoop.log.dir/,STOPPED} 2024-11-25T05:58:08,227 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-25T05:58:08,348 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down